From 8db36b7b2ed0dff3a5baea3d2a41b5f1a14f58d6 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:27:22 +0300 Subject: [PATCH 01/11] refactor(p2p/ipld): move-in DAHeader Share --- p2p/ipld/header.go | 146 ++++++++++++++++++++++++++++++++++++++++ p2p/ipld/header_test.go | 17 +++++ p2p/ipld/share.go | 51 ++++++++++++++ 3 files changed, 214 insertions(+) create mode 100644 p2p/ipld/header.go create mode 100644 p2p/ipld/header_test.go create mode 100644 p2p/ipld/share.go diff --git a/p2p/ipld/header.go b/p2p/ipld/header.go new file mode 100644 index 0000000000..67f4530a33 --- /dev/null +++ b/p2p/ipld/header.go @@ -0,0 +1,146 @@ +package ipld + +import ( + "bytes" + "errors" + "fmt" + + "github.com/lazyledger/nmt/namespace" + "github.com/lazyledger/rsmt2d" + + "github.com/lazyledger/lazyledger-core/crypto/merkle" + "github.com/lazyledger/lazyledger-core/proto/tendermint/types" +) + +// DataAvailabilityHeader (DAHeader) contains the row and column roots of the erasure +// coded version of the data in Block.Data. +// Therefor the original Block.Data is arranged in a +// k × k matrix, which is then "extended" to a +// 2k × 2k matrix applying multiple times Reed-Solomon encoding. +// For details see Section 5.2: https://arxiv.org/abs/1809.09044 +// or the LazyLedger specification: +// https://github.com/lazyledger/lazyledger-specs/blob/master/specs/data_structures.md#availabledataheader +// Note that currently we list row and column roots in separate fields +// (different from the spec). +type DataAvailabilityHeader struct { + // RowRoot_j = root((M_{j,1} || M_{j,2} || ... || M_{j,2k} )) + RowsRoots NmtRoots `json:"row_roots"` + // ColumnRoot_j = root((M_{1,j} || M_{2,j} || ... || M_{2k,j} )) + ColumnRoots NmtRoots `json:"column_roots"` + // cached result of Hash() not to be recomputed + hash []byte +} + +// TODO(Wondertan): Should return single Hash/CID instead +func MakeDataHeader(eds *rsmt2d.ExtendedDataSquare) *DataAvailabilityHeader { + // generate the row and col roots using the EDS and nmt wrapper + rowRoots, colRoots := eds.RowRoots(), eds.ColumnRoots() + // create DAH + dah := &DataAvailabilityHeader{ + RowsRoots: make([]namespace.IntervalDigest, eds.Width()), + ColumnRoots: make([]namespace.IntervalDigest, eds.Width()), + } + // todo(evan): remove interval digests + // convert the roots to interval digests + for i := 0; i < len(rowRoots); i++ { + rowRoot, err := namespace.IntervalDigestFromBytes(NamespaceSize, rowRoots[i]) + if err != nil { + panic(err) + } + colRoot, err := namespace.IntervalDigestFromBytes(NamespaceSize, colRoots[i]) + if err != nil { + panic(err) + } + dah.RowsRoots[i] = rowRoot + dah.ColumnRoots[i] = colRoot + } + return dah +} + +type NmtRoots []namespace.IntervalDigest + +func (roots NmtRoots) Bytes() [][]byte { + res := make([][]byte, len(roots)) + for i := 0; i < len(roots); i++ { + res[i] = roots[i].Bytes() + } + return res +} + +func NmtRootsFromBytes(in [][]byte) (roots NmtRoots, err error) { + roots = make([]namespace.IntervalDigest, len(in)) + for i := 0; i < len(in); i++ { + roots[i], err = namespace.IntervalDigestFromBytes(NamespaceSize, in[i]) + if err != nil { + return roots, err + } + } + return +} + +// String returns hex representation of merkle hash of the DAHeader. +func (dah *DataAvailabilityHeader) String() string { + if dah == nil { + return "" + } + return fmt.Sprintf("%X", dah.Hash()) +} + +// Equals checks equality of two DAHeaders. +func (dah *DataAvailabilityHeader) Equals(to *DataAvailabilityHeader) bool { + return bytes.Equal(dah.Hash(), to.Hash()) +} + +// Hash computes and caches the merkle root of the row and column roots. +func (dah *DataAvailabilityHeader) Hash() []byte { + if dah == nil { + return merkle.HashFromByteSlices(nil) + } + if len(dah.hash) != 0 { + return dah.hash + } + + colsCount := len(dah.ColumnRoots) + rowsCount := len(dah.RowsRoots) + slices := make([][]byte, colsCount+rowsCount) + for i, rowRoot := range dah.RowsRoots { + slices[i] = rowRoot.Bytes() + } + for i, colRoot := range dah.ColumnRoots { + slices[i+colsCount] = colRoot.Bytes() + } + // The single data root is computed using a simple binary merkle tree. + // Effectively being root(rowRoots || columnRoots): + dah.hash = merkle.HashFromByteSlices(slices) + return dah.hash +} + +func (dah *DataAvailabilityHeader) ToProto() (*types.DataAvailabilityHeader, error) { + if dah == nil { + return nil, errors.New("nil DataAvailabilityHeader") + } + + dahp := new(types.DataAvailabilityHeader) + dahp.RowRoots = dah.RowsRoots.Bytes() + dahp.ColumnRoots = dah.ColumnRoots.Bytes() + return dahp, nil +} + +func DataAvailabilityHeaderFromProto(dahp *types.DataAvailabilityHeader) (dah *DataAvailabilityHeader, err error) { + if dahp == nil { + return nil, errors.New("nil DataAvailabilityHeader") + } + + dah = new(DataAvailabilityHeader) + dah.RowsRoots, err = NmtRootsFromBytes(dahp.RowRoots) + if err != nil { + return + } + + dah.ColumnRoots, err = NmtRootsFromBytes(dahp.ColumnRoots) + if err != nil { + return + } + + return +} diff --git a/p2p/ipld/header_test.go b/p2p/ipld/header_test.go new file mode 100644 index 0000000000..a925221349 --- /dev/null +++ b/p2p/ipld/header_test.go @@ -0,0 +1,17 @@ +package ipld + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// This follows RFC-6962, i.e. `echo -n '' | sha256sum` +var emptyBytes = []byte{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, + 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, + 0x78, 0x52, 0xb8, 0x55} + +func TestNilDataAvailabilityHeaderHashDoesntCrash(t *testing.T) { + assert.Equal(t, emptyBytes, (*DataAvailabilityHeader)(nil).Hash()) + assert.Equal(t, emptyBytes, new(DataAvailabilityHeader).Hash()) +} diff --git a/p2p/ipld/share.go b/p2p/ipld/share.go new file mode 100644 index 0000000000..65ada35b20 --- /dev/null +++ b/p2p/ipld/share.go @@ -0,0 +1,51 @@ +package ipld + +import "github.com/lazyledger/nmt/namespace" + +const ( + // MaxSquareSize is currently the maximum size supported for unerasured data in rsmt2d.ExtendedDataSquare. + MaxSquareSize = 128 + // ShareSize system wide default size for data shares. + ShareSize = 256 + // NamespaceSize is a system wide size for NMT namespaces. + // TODO(Wondertan): Should be part of IPLD/NMT plugin + NamespaceSize = 8 +) + +// TODO(Wondertan): +// Currently Share prepends namespace bytes while NamespaceShare just takes a copy of namespace +// separating it in separate field. This is really confusing for newcomers and even for those who worked with code, +// but had some time off of it. Instead, we shouldn't copy(1) and likely have only one type - NamespacedShare, as we +// don't support shares without namespace. + +// Share contains the raw share data without the corresponding namespace. +type Share []byte + +// TODO(Wondertan): Consider using alias to namespace.PrefixedData instead +// NamespacedShare extends a Share with the corresponding namespace. +type NamespacedShare struct { + Share + ID namespace.ID +} + +func (n NamespacedShare) NamespaceID() namespace.ID { + return n.ID +} + +func (n NamespacedShare) Data() []byte { + return n.Share +} + +// NamespacedShares is just a list of NamespacedShare elements. +// It can be used to extract the raw shares. +type NamespacedShares []NamespacedShare + +// Raw returns the raw shares that can be fed into the erasure coding +// library (e.g. rsmt2d). +func (ns NamespacedShares) Raw() [][]byte { + res := make([][]byte, len(ns)) + for i, nsh := range ns { + res[i] = nsh.Share + } + return res +} From c21909f1f66c789026dcfd781d93c87103676803 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:30:09 +0300 Subject: [PATCH 02/11] refactor(p2p/ipld): rework PutBlock API to PutData --- p2p/ipld/read.go | 54 ++++++++++++++++-------------------------------- 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/p2p/ipld/read.go b/p2p/ipld/read.go index 45fa5d4986..ef3e3155c0 100644 --- a/p2p/ipld/read.go +++ b/p2p/ipld/read.go @@ -2,6 +2,7 @@ package ipld import ( "context" + "errors" "fmt" "math/rand" @@ -11,67 +12,46 @@ import ( "github.com/lazyledger/lazyledger-core/ipfs/plugin" "github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper" - "github.com/lazyledger/lazyledger-core/types" - "github.com/lazyledger/lazyledger-core/types/consts" ) -const baseErrorMsg = "failure to retrieve block data:" +var ErrRetrieveTimeout = errors.New("retrieve data timeout") -var ErrEncounteredTooManyErrors = fmt.Errorf("%s %s", baseErrorMsg, "encountered too many errors") -var ErrTimeout = fmt.Errorf("%s %s", baseErrorMsg, "timeout") - -// RetrieveBlockData asynchronously fetches block data using the minimum number +// RetrieveData asynchronously fetches block data using the minimum number // of requests to IPFS. It fails if one of the random samples sampled is not available. -func RetrieveBlockData( +func RetrieveData( ctx context.Context, - dah *types.DataAvailabilityHeader, + dah *DataAvailabilityHeader, dag ipld.NodeGetter, codec rsmt2d.Codec, -) (types.Data, error) { +) (*rsmt2d.ExtendedDataSquare, error) { edsWidth := len(dah.RowsRoots) sc := newshareCounter(ctx, uint32(edsWidth)) - // convert the row and col roots into Cids rowRoots := dah.RowsRoots.Bytes() colRoots := dah.ColumnRoots.Bytes() - // sample 1/4 of the total extended square by sampling half of the leaves in // half of the rows for _, row := range uniqueRandNumbers(edsWidth/2, edsWidth) { for _, col := range uniqueRandNumbers(edsWidth/2, edsWidth) { rootCid, err := plugin.CidFromNamespacedSha256(rowRoots[row]) if err != nil { - return types.Data{}, err + return nil, err } go sc.retrieveShare(rootCid, true, row, col, dag) } } - // wait until enough data has been collected, too many errors encountered, // or the timeout is reached err := sc.wait() if err != nil { - return types.Data{}, err + return nil, err } - // flatten the square flattened := sc.flatten() - - tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(edsWidth) / 2) - // repair the square - eds, err := rsmt2d.RepairExtendedDataSquare(rowRoots, colRoots, flattened, codec, tree.Constructor) - if err != nil { - return types.Data{}, err - } - - blockData, err := types.DataFromSquare(eds) - if err != nil { - return types.Data{}, err - } - - return blockData, nil + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(edsWidth) / 2) + return rsmt2d.RepairExtendedDataSquare(rowRoots, colRoots, flattened, codec, tree.Constructor) } // uniqueRandNumbers generates count unique random numbers with a max of max @@ -136,7 +116,7 @@ func newshareCounter(parentCtx context.Context, edsWidth uint32) *shareCounter { shares: make(map[index][]byte), edsWidth: edsWidth, minSharesNeeded: minSharesNeeded, - shareChan: make(chan indexedShare, 1), + shareChan: make(chan indexedShare, 512), errc: make(chan error, 1), ctx: ctx, cancel: cancel, @@ -159,7 +139,7 @@ func (sc *shareCounter) retrieveShare( } } - if len(data) < consts.ShareSize { + if len(data) < ShareSize { return } @@ -173,8 +153,7 @@ func (sc *shareCounter) retrieveShare( select { case <-sc.ctx.Done(): - default: - sc.shareChan <- indexedShare{data: data[consts.NamespaceSize:], index: index{row: rowIdx, col: colIdx}} + case sc.shareChan <- indexedShare{data: data[NamespaceSize:], index: index{row: rowIdx, col: colIdx}}: } } @@ -186,8 +165,11 @@ func (sc *shareCounter) wait() error { for { select { case <-sc.ctx.Done(): - return ErrTimeout - + err := sc.ctx.Err() + if err == context.DeadlineExceeded { + return ErrRetrieveTimeout + } + return err case share := <-sc.shareChan: _, has := sc.shares[share.index] // add iff it does not already exists From 6fb48fd8e2e719f8742daa6abdeb7048b4e63e45 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:31:19 +0300 Subject: [PATCH 03/11] refactor(p2p/ipld): rework RetrieveData to the latest changes --- p2p/ipld/write.go | 63 ++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/p2p/ipld/write.go b/p2p/ipld/write.go index 44fa5dc11b..f9982d32a2 100644 --- a/p2p/ipld/write.go +++ b/p2p/ipld/write.go @@ -18,57 +18,47 @@ import ( "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/libs/sync" "github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper" - "github.com/lazyledger/lazyledger-core/types" ) -// PutBlock posts and pins erasured block data to IPFS using the provided -// ipld.NodeAdder. Note: the erasured data is currently recomputed -// TODO this craves for refactor -func PutBlock( - ctx context.Context, - adder ipld.NodeAdder, - block *types.Block, - croute routing.ContentRouting, - logger log.Logger, -) error { - // recompute the shares - namespacedShares, _ := block.Data.ComputeShares() - shares := namespacedShares.RawShares() +// TODO(Wondertan) Improve API - // don't do anything if there is no data to put on IPFS +// PutData posts erasured block data to IPFS using the provided ipld.NodeAdder. +func PutData(ctx context.Context, shares NamespacedShares, adder ipld.NodeAdder) (*rsmt2d.ExtendedDataSquare, error) { if len(shares) == 0 { - return nil + return nil, fmt.Errorf("empty data") // empty block is not an empty Data } - // create nmt adder wrapping batch adder batchAdder := NewNmtNodeAdder(ctx, ipld.NewBatch(ctx, adder)) - // create the nmt wrapper to generate row and col commitments squareSize := uint32(math.Sqrt(float64(len(shares)))) tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize), nmt.NodeVisitor(batchAdder.Visit)) - // recompute the eds - eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), tree.Constructor) + eds, err := rsmt2d.ComputeExtendedDataSquare(shares.Raw(), rsmt2d.NewRSGF8Codec(), tree.Constructor) if err != nil { - return fmt.Errorf("failure to recompute the extended data square: %w", err) + return nil, fmt.Errorf("failure to recompute the extended data square: %w", err) } + // compute roots + eds.ColumnRoots() + // commit the batch to ipfs + return eds, batchAdder.Commit() +} + +func ProvideData( + ctx context.Context, + dah *DataAvailabilityHeader, + croute routing.ContentRouting, + logger log.Logger) error { // get row and col roots to be provided // this also triggers adding data to DAG - prov := newProvider(ctx, croute, int32(squareSize*4), logger.With("height", block.Height)) - for _, root := range eds.RowRoots() { - prov.Provide(plugin.MustCidFromNamespacedSha256(root)) + prov := newProvider(ctx, croute, int32(len(dah.RowsRoots)+len(dah.ColumnRoots)), logger) + for _, root := range dah.RowsRoots { + prov.Provide(plugin.MustCidFromNamespacedSha256(root.Bytes())) } - for _, root := range eds.ColumnRoots() { - prov.Provide(plugin.MustCidFromNamespacedSha256(root)) - } - // commit the batch to ipfs - err = batchAdder.Commit() - if err != nil { - return err + for _, root := range dah.ColumnRoots { + prov.Provide(plugin.MustCidFromNamespacedSha256(root.Bytes())) } // wait until we provided all the roots if requested - <-prov.Done() - return prov.Err() + return prov.Done() } var provideWorkers = 32 @@ -112,8 +102,9 @@ func (p *provider) Provide(id cid.Cid) { } } -func (p *provider) Done() <-chan struct{} { - return p.done +func (p *provider) Done() error { + <-p.done + return p.Err() } func (p *provider) Err() error { @@ -139,7 +130,7 @@ func (p *provider) worker() { p.errLk.Unlock() } - p.log.Error("failed to provide to DHT", "err", err.Error()) + p.log.Error("Failed to provide to DHT", "err", err.Error()) } p.provided() From aeda818b91ff4ac3928afc51ad9af06bd0dd6126 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:31:46 +0300 Subject: [PATCH 04/11] refactor(p2p/ipld): rework ValidateAvailability to the latest changes --- p2p/ipld/validate.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/p2p/ipld/validate.go b/p2p/ipld/validate.go index 94cdab066f..8f278b2644 100644 --- a/p2p/ipld/validate.go +++ b/p2p/ipld/validate.go @@ -7,9 +7,6 @@ import ( "time" ipld "github.com/ipfs/go-ipld-format" - "github.com/lazyledger/nmt/namespace" - - "github.com/lazyledger/lazyledger-core/types" ) // ValidationTimeout specifies timeout for DA validation during which data have to be found on the network, @@ -28,9 +25,9 @@ var ErrValidationFailed = errors.New("validation failed") func ValidateAvailability( ctx context.Context, dag ipld.NodeGetter, - dah *types.DataAvailabilityHeader, + dah *DataAvailabilityHeader, numSamples int, - onLeafValidity func(namespace.PrefixedData8), + onLeafValidity func(NamespacedShare), ) error { ctx, cancel := context.WithTimeout(ctx, ValidationTimeout) defer cancel() @@ -75,7 +72,7 @@ func ValidateAvailability( // the fact that we read the data, already gives us Merkle proof, // thus the data availability is successfully validated :) - onLeafValidity(r.data) + onLeafValidity(NamespacedShare{ID: r.data[:NamespaceSize], Share: r.data}) case <-ctx.Done(): err := ctx.Err() if err == context.DeadlineExceeded { From 730fdc33835e63be6df48df3189d03fe25a6871f Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:33:28 +0300 Subject: [PATCH 05/11] refactor(p2p/ipld): rework tests and test utilities to the latest API changes --- go.mod | 1 + go.sum | 1 - ipfs/plugin/nmt.go | 2 +- p2p/ipld/net_test.go | 85 +++++++----- p2p/ipld/read_test.go | 192 +++++--------------------- p2p/ipld/sample.go | 3 +- p2p/ipld/testing.go | 87 ++++++++++++ p2p/ipld/validate_test.go | 25 +--- p2p/ipld/write_test.go | 284 ++++++++++++++++---------------------- 9 files changed, 305 insertions(+), 375 deletions(-) create mode 100644 p2p/ipld/testing.go diff --git a/go.mod b/go.mod index 574ddb2401..d010ed9319 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c google.golang.org/grpc v1.39.0 ) diff --git a/go.sum b/go.sum index df002a1f8a..c8a6674d03 100644 --- a/go.sum +++ b/go.sum @@ -1824,7 +1824,6 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= diff --git a/ipfs/plugin/nmt.go b/ipfs/plugin/nmt.go index f42d15bf6b..48b962189b 100644 --- a/ipfs/plugin/nmt.go +++ b/ipfs/plugin/nmt.go @@ -380,7 +380,7 @@ func (l nmtLeafNode) Size() (uint64, error) { return 0, nil } -// CidFromNamespacedSha256 uses a hash from an nmt tree to create a cide +// CidFromNamespacedSha256 uses a hash from an nmt tree to create a CID func CidFromNamespacedSha256(namespacedHash []byte) (cid.Cid, error) { if got, want := len(namespacedHash), nmtHashSize; got != want { return cid.Cid{}, fmt.Errorf("invalid namespaced hash length, got: %v, want: %v", got, want) diff --git a/p2p/ipld/net_test.go b/p2p/ipld/net_test.go index 4e20652bf2..1019ccc738 100644 --- a/p2p/ipld/net_test.go +++ b/p2p/ipld/net_test.go @@ -18,11 +18,9 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" - "github.com/lazyledger/lazyledger-core/ipfs/plugin" "github.com/lazyledger/lazyledger-core/libs/log" - "github.com/lazyledger/lazyledger-core/types" - "github.com/lazyledger/lazyledger-core/types/consts" ) func TestDiscovery(t *testing.T) { @@ -32,17 +30,8 @@ func TestDiscovery(t *testing.T) { dhts := dhtNet(ctx, t, 2) dht1, dht2 := dhts[0], dhts[0] - data := generateRandomBlockData(64, consts.MsgShareSize-2) - b := &types.Block{ - Data: data, - LastCommit: &types.Commit{}, - } - b.Hash() - - id, err := plugin.CidFromNamespacedSha256(b.DataAvailabilityHeader.RowsRoots[0].Bytes()) - require.NoError(t, err) - - err = dht1.Provide(ctx, id, false) + id := RandNamespacedCID(t) + err := dht1.Provide(ctx, id, false) require.NoError(t, err) prvs, err := dht2.FindProviders(ctx, id) @@ -50,38 +39,70 @@ func TestDiscovery(t *testing.T) { assert.Equal(t, dht1.PeerID(), prvs[0].ID, "peer not found") } -func TestWriteDiscoveryReadData(t *testing.T) { - logger := log.TestingLogger() +func TestWriteDiscoveryValidateReadData(t *testing.T) { + const ( + netSize = 4 + edsSize = 4 // TODO(Wondertan): Increase size once race issue is fixed + samples = 16 + ) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - dags, dhts := dagNet(ctx, t, 5) - blocks := make([]*types.Block, len(dags)) - for i, dag := range dags { - data := generateRandomBlockData(64, consts.MsgShareSize-2) - b := &types.Block{ - Data: data, - LastCommit: &types.Commit{}, - } - b.Hash() - blocks[i] = b + logger := log.TestingLogger() + dags, dhts := dagNet(ctx, t, netSize) - err := PutBlock(ctx, dag, blocks[i], dhts[i], logger) - require.NoError(t, err) + gp, execCtx := errgroup.WithContext(ctx) + eds := make([]*rsmt2d.ExtendedDataSquare, len(dags)) + for i, dag := range dags { + i, dag := i, dag + gp.Go(func() (err error) { + eds[i], err = PutData(execCtx, RandNamespacedShares(t, edsSize*edsSize), dag) + if err != nil { + return + } + return ProvideData(execCtx, MakeDataHeader(eds[i]), dhts[i], logger) + }) } + err := gp.Wait() + require.NoError(t, err) + gp, execCtx = errgroup.WithContext(ctx) for i, dag := range dags { if i == len(dags)-1 { i = 0 } + i, dag := i, dag + gp.Go(func() error { + exp := eds[i+1] + return ValidateAvailability(execCtx, dag, MakeDataHeader(exp), samples, func(NamespacedShare) {}) + }) + } + err = gp.Wait() + require.NoError(t, err) - exp := blocks[i+1] - actual, err := RetrieveBlockData(ctx, &exp.DataAvailabilityHeader, dag, rsmt2d.NewRSGF8Codec()) - assert.NoError(t, err) - assert.EqualValues(t, exp.Data.Txs, actual.Txs, "blocks are not equal") + gp, execCtx = errgroup.WithContext(ctx) + for i, dag := range dags { + if i == len(dags)-1 { + i = 0 + } + i, dag := i, dag + gp.Go(func() error { + exp := eds[i+1] + got, err := RetrieveData(execCtx, MakeDataHeader(exp), dag, rsmt2d.NewRSGF8Codec()) + if err != nil { + return err + } + assert.True(t, EqualEDS(exp, got)) + return nil + }) } + err = gp.Wait() + require.NoError(t, err) } +// TODO(Wondertan): Consider making utilities below as public + func dagNet(ctx context.Context, t *testing.T, num int) ([]ipld.DAGService, []*dht.IpfsDHT) { net := mocknet.New(ctx) _, medium := dagNode(ctx, t, net) diff --git a/p2p/ipld/read_test.go b/p2p/ipld/read_test.go index 690caf321c..8dc52d5107 100644 --- a/p2p/ipld/read_test.go +++ b/p2p/ipld/read_test.go @@ -4,27 +4,21 @@ import ( "bytes" "context" "crypto/sha256" - "fmt" "math" "math/rand" - "sort" "testing" "time" + "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" mdutils "github.com/ipfs/go-merkledag/test" "github.com/lazyledger/nmt" - "github.com/lazyledger/nmt/namespace" "github.com/lazyledger/rsmt2d" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/lazyledger/lazyledger-core/ipfs" "github.com/lazyledger/lazyledger-core/ipfs/plugin" - "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper" - "github.com/lazyledger/lazyledger-core/types" - "github.com/lazyledger/lazyledger-core/types/consts" ) func TestGetLeafData(t *testing.T) { @@ -32,23 +26,19 @@ func TestGetLeafData(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() + dag := mdutils.Mock() - // generate random data for the nmt - data := generateRandNamespacedRawData(leaves, consts.NamespaceSize, consts.ShareSize) + // generate random shares for the nmt + shares := RandNamespacedShares(t, leaves) // create a random tree - dag := mdutils.Mock() - root, err := getNmtRoot(ctx, dag, data) - require.NoError(t, err) - - // compute the root and create a cid for the root hash - rootCid, err := plugin.CidFromNamespacedSha256(root.Bytes()) + root, err := getNmtRoot(ctx, dag, shares.Raw()) require.NoError(t, err) - for i, leaf := range data { - data, err := GetLeafData(ctx, rootCid, uint32(i), uint32(len(data)), dag) - assert.NoError(t, err) - assert.Equal(t, leaf, data) + for i, leaf := range shares { + data, err := GetLeafData(ctx, root, uint32(i), uint32(len(shares)), dag) + require.NoError(t, err) + assert.True(t, bytes.Equal(leaf.Share, data)) } } @@ -59,12 +49,12 @@ func TestBlockRecovery(t *testing.T) { extendedShareCount := extendedSquareWidth * extendedSquareWidth // generate test data - quarterShares := generateRandNamespacedRawData(shareCount, consts.NamespaceSize, consts.MsgShareSize) - allShares := generateRandNamespacedRawData(shareCount, consts.NamespaceSize, consts.MsgShareSize) + quarterShares := RandNamespacedShares(t, shareCount) + allShares := RandNamespacedShares(t, shareCount) testCases := []struct { name string - shares [][]byte + shares NamespacedShares expectErr bool errString string d int // number of shares to delete @@ -84,7 +74,7 @@ func TestBlockRecovery(t *testing.T) { tree := wrapper.NewErasuredNamespacedMerkleTree(squareSize) recoverTree := wrapper.NewErasuredNamespacedMerkleTree(squareSize) - eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.NewRSGF8Codec(), tree.Constructor) + eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares.Raw(), rsmt2d.NewRSGF8Codec(), tree.Constructor) require.NoError(t, err) // calculate roots using the first complete square @@ -116,21 +106,19 @@ func TestBlockRecovery(t *testing.T) { } func TestRetrieveBlockData(t *testing.T) { - logger := log.TestingLogger() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dag := mdutils.Mock() + type test struct { name string squareSize int - expectErr bool - errStr string } tests := []test{ - {"Empty block", 1, false, ""}, - {"4 KB block", 4, false, ""}, - {"16 KB block", 8, false, ""}, - {"16 KB block timeout expected", 8, true, "not found"}, - {"max square size", consts.MaxSquareSize, false, ""}, + {"1x1(min)", 1}, + {"32x32(med)", 32}, + {"128x128(max)", MaxSquareSize}, } - for _, tc := range tests { // TODO(Wondertan): remove this if tc.squareSize > 8 { @@ -138,58 +126,18 @@ func TestRetrieveBlockData(t *testing.T) { } tc := tc - t.Run(fmt.Sprintf("%s size %d", tc.name, tc.squareSize), func(t *testing.T) { - ctx := context.Background() - dag := mdutils.Mock() - croute := ipfs.MockRouting() - - blockData := generateRandomBlockData(tc.squareSize*tc.squareSize, consts.MsgShareSize-2) - block := &types.Block{ - Data: blockData, - LastCommit: &types.Commit{}, - } - - // if an error is exected, don't put the block - if !tc.expectErr { - err := PutBlock(ctx, dag, block, croute, logger) - require.NoError(t, err) - } - - shareData, _ := blockData.ComputeShares() - rawData := shareData.RawShares() - - tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(tc.squareSize)) - eds, err := rsmt2d.ComputeExtendedDataSquare(rawData, rsmt2d.NewRSGF8Codec(), tree.Constructor) + t.Run(tc.name, func(t *testing.T) { + shares := RandNamespacedShares(t, tc.squareSize*tc.squareSize) + in, err := PutData(ctx, shares, dag) require.NoError(t, err) - rawRowRoots := eds.RowRoots() - rawColRoots := eds.ColumnRoots() - rowRoots := rootsToDigests(rawRowRoots) - colRoots := rootsToDigests(rawColRoots) - - // limit with deadline retrieval specifically + // limit with deadline, specifically retrieval ctx, cancel := context.WithTimeout(ctx, time.Second*2) defer cancel() - rblockData, err := RetrieveBlockData( - ctx, - &types.DataAvailabilityHeader{ - RowsRoots: rowRoots, - ColumnRoots: colRoots, - }, - dag, - rsmt2d.NewRSGF8Codec(), - ) - - if tc.expectErr { - require.Error(t, err) - require.Contains(t, err.Error(), tc.errStr) - return - } + out, err := RetrieveData(ctx, MakeDataHeader(in), dag, rsmt2d.NewRSGF8Codec()) require.NoError(t, err) - - nsShares, _ := rblockData.ComputeShares() - assert.Equal(t, rawData, nsShares.RawShares()) + assert.True(t, EqualEDS(in, out)) }) } } @@ -212,46 +160,23 @@ func getNmtRoot( ctx context.Context, dag format.NodeAdder, namespacedData [][]byte, -) (namespace.IntervalDigest, error) { +) (cid.Cid, error) { na := NewNmtNodeAdder(ctx, format.NewBatch(ctx, dag)) - tree := nmt.New(sha256.New, nmt.NamespaceIDSize(consts.NamespaceSize), nmt.NodeVisitor(na.Visit)) + tree := nmt.New(sha256.New, nmt.NamespaceIDSize(NamespaceSize), nmt.NodeVisitor(na.Visit)) for _, leaf := range namespacedData { err := tree.Push(leaf) if err != nil { - return namespace.IntervalDigest{}, err - } - } - - return tree.Root(), na.Commit() -} - -// this code is copy pasted from the plugin, and should likely be exported in the plugin instead -func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte { - data := make([][]byte, total) - for i := 0; i < total; i++ { - nid := make([]byte, nidSize) - _, err := rand.Read(nid) - if err != nil { - panic(err) + return cid.Undef, err } - data[i] = nid } - sortByteArrays(data) - for i := 0; i < total; i++ { - d := make([]byte, leafSize) - _, err := rand.Read(d) - if err != nil { - panic(err) - } - data[i] = append(data[i], d...) + // call Root early as it initiates saving + root := tree.Root() + if err := na.Commit(); err != nil { + return cid.Undef, err } - return data -} - -func sortByteArrays(src [][]byte) { - sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 }) + return plugin.CidFromNamespacedSha256(root.Bytes()) } // removes d shares from data @@ -268,52 +193,3 @@ func removeRandShares(data [][]byte, d int) [][]byte { } return data } - -func rootsToDigests(roots [][]byte) []namespace.IntervalDigest { - out := make([]namespace.IntervalDigest, len(roots)) - for i, root := range roots { - idigest, err := namespace.IntervalDigestFromBytes(consts.NamespaceSize, root) - if err != nil { - panic(err) - } - out[i] = idigest - } - return out -} - -func generateRandomBlockData(msgCount, msgSize int) types.Data { - var out types.Data - if msgCount == 1 { - return out - } - out.Messages = generateRandomMessages(msgCount-1, msgSize) - out.Txs = generateRandomContiguousShares(1) - return out -} - -func generateRandomMessages(count, msgSize int) types.Messages { - shares := generateRandNamespacedRawData(count, consts.NamespaceSize, msgSize) - msgs := make([]types.Message, count) - for i, s := range shares { - msgs[i] = types.Message{ - Data: s[consts.NamespaceSize:], - NamespaceID: s[:consts.NamespaceSize], - } - } - return types.Messages{MessagesList: msgs} -} - -func generateRandomContiguousShares(count int) types.Txs { - // the size of a length delimited tx that takes up an entire share - const adjustedTxSize = consts.TxShareSize - 2 - txs := make(types.Txs, count) - for i := 0; i < count; i++ { - tx := make([]byte, adjustedTxSize) - _, err := rand.Read(tx) - if err != nil { - panic(err) - } - txs[i] = types.Tx(tx) - } - return txs -} diff --git a/p2p/ipld/sample.go b/p2p/ipld/sample.go index 9777cb615d..42650ebc63 100644 --- a/p2p/ipld/sample.go +++ b/p2p/ipld/sample.go @@ -8,7 +8,6 @@ import ( "github.com/lazyledger/nmt/namespace" "github.com/lazyledger/lazyledger-core/ipfs/plugin" - "github.com/lazyledger/lazyledger-core/types" ) // Sample is a point in 2D space over square. @@ -25,7 +24,7 @@ func SampleSquare(squareWidth uint32, num int) []Sample { } // Leaf returns leaf info needed for retrieval using data provided with DAHeader. -func (s Sample) Leaf(dah *types.DataAvailabilityHeader) (cid.Cid, uint32, error) { +func (s Sample) Leaf(dah *DataAvailabilityHeader) (cid.Cid, uint32, error) { var ( leaf uint32 root namespace.IntervalDigest diff --git a/p2p/ipld/testing.go b/p2p/ipld/testing.go new file mode 100644 index 0000000000..7a864fc5e7 --- /dev/null +++ b/p2p/ipld/testing.go @@ -0,0 +1,87 @@ +package ipld + +import ( + "bytes" + "crypto/sha256" + "math" + mrand "math/rand" + "sort" + "testing" + + "github.com/ipfs/go-cid" + "github.com/lazyledger/rsmt2d" + "github.com/stretchr/testify/require" + + "github.com/lazyledger/lazyledger-core/ipfs/plugin" + "github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper" +) + +// TODO(Wondertan): Move to rsmt2d +// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and Golang practises +func EqualEDS(a *rsmt2d.ExtendedDataSquare, b *rsmt2d.ExtendedDataSquare) bool { + if a.Width() != b.Width() { + return false + } + + for i := uint(0); i < a.Width(); i++ { + ar, br := a.Row(i), b.Row(i) + for j := 0; j < len(ar); j++ { + if !bytes.Equal(ar[j], br[j]) { + return false + } + } + } + + return true +} + +// TODO(Wondertan): Move to NMT plugin +func RandNamespacedCID(t *testing.T) cid.Cid { + raw := make([]byte, NamespaceSize*2+sha256.Size) + _, err := mrand.Read(raw) // nolint:gosec // G404: Use of weak random number generator + require.NoError(t, err) + id, err := plugin.CidFromNamespacedSha256(raw) + require.NoError(t, err) + return id +} + +func RandEDS(t *testing.T, size int) *rsmt2d.ExtendedDataSquare { + shares := RandNamespacedShares(t, size*size) + // create the nmt wrapper to generate row and col commitments + squareSize := uint32(math.Sqrt(float64(len(shares)))) + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize)) + // recompute the eds + eds, err := rsmt2d.ComputeExtendedDataSquare(shares.Raw(), rsmt2d.NewRSGF8Codec(), tree.Constructor) + require.NoError(t, err, "failure to recompute the extended data square") + return eds +} + +func RandNamespacedShares(t *testing.T, total int) NamespacedShares { + if total&(total-1) != 0 { + t.Fatal("Namespace total must be power of 2") + } + + data := make([][]byte, total) + for i := 0; i < total; i++ { + nid := make([]byte, NamespaceSize) + _, err := mrand.Read(nid) // nolint:gosec // G404: Use of weak random number generator + require.NoError(t, err) + data[i] = nid + } + sortByteArrays(data) + + shares := make(NamespacedShares, total) + for i := 0; i < total; i++ { + shares[i].ID = data[i] + shares[i].Share = make([]byte, NamespaceSize+ShareSize) + copy(shares[i].Share[:NamespaceSize], data[i]) + _, err := mrand.Read(shares[i].Share[NamespaceSize:]) // nolint:gosec // G404: Use of weak random number generator + require.NoError(t, err) + } + + return shares +} + +func sortByteArrays(src [][]byte) { + sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 }) +} diff --git a/p2p/ipld/validate_test.go b/p2p/ipld/validate_test.go index c7e231e732..14260f97e0 100644 --- a/p2p/ipld/validate_test.go +++ b/p2p/ipld/validate_test.go @@ -6,41 +6,28 @@ import ( "time" mdutils "github.com/ipfs/go-merkledag/test" - "github.com/lazyledger/nmt/namespace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/lazyledger/lazyledger-core/ipfs" - "github.com/lazyledger/lazyledger-core/libs/log" - "github.com/lazyledger/lazyledger-core/types" - "github.com/lazyledger/lazyledger-core/types/consts" ) // TODO(@Wondertan): Add test to simulate ErrValidationFailed func TestValidateAvailability(t *testing.T) { const ( - shares = 15 - squareSize = 8 - adjustedMsgSize = consts.MsgShareSize - 2 + shares = 16 + squareSize = 8 * 8 ) + dag := mdutils.Mock() ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - blockData := generateRandomBlockData(squareSize*squareSize, adjustedMsgSize) - block := &types.Block{ - Data: blockData, - LastCommit: &types.Commit{}, - } - block.Hash() - - dag := mdutils.Mock() - err := PutBlock(ctx, dag, block, ipfs.MockRouting(), log.TestingLogger()) + blockData := RandNamespacedShares(t, squareSize) + eds, err := PutData(ctx, blockData, dag) require.NoError(t, err) calls := 0 - err = ValidateAvailability(ctx, dag, &block.DataAvailabilityHeader, shares, func(data namespace.PrefixedData8) { + err = ValidateAvailability(ctx, dag, MakeDataHeader(eds), shares, func(share NamespacedShare) { calls++ }) assert.NoError(t, err) diff --git a/p2p/ipld/write_test.go b/p2p/ipld/write_test.go index d5e37faf6a..e8b7d7cad2 100644 --- a/p2p/ipld/write_test.go +++ b/p2p/ipld/write_test.go @@ -2,196 +2,156 @@ package ipld import ( "context" - mrand "math/rand" "testing" "time" mdutils "github.com/ipfs/go-merkledag/test" - "github.com/lazyledger/nmt" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - abci "github.com/lazyledger/lazyledger-core/abci/types" - "github.com/lazyledger/lazyledger-core/crypto/tmhash" - "github.com/lazyledger/lazyledger-core/ipfs" "github.com/lazyledger/lazyledger-core/ipfs/plugin" - "github.com/lazyledger/lazyledger-core/libs/log" - tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" - "github.com/lazyledger/lazyledger-core/types" - "github.com/lazyledger/lazyledger-core/types/consts" ) func TestPutBlock(t *testing.T) { - logger := log.TestingLogger() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dag := mdutils.Mock() - croute := ipfs.MockRouting() - - maxOriginalSquareSize := consts.MaxSquareSize / 2 - maxShareCount := maxOriginalSquareSize * maxOriginalSquareSize testCases := []struct { - name string - blockData types.Data - expectErr bool - errString string + name string + squareSize int }{ - {"no leaves", generateRandomMsgOnlyData(0), false, ""}, - {"single leaf", generateRandomMsgOnlyData(1), false, ""}, - {"16 leaves", generateRandomMsgOnlyData(16), false, ""}, - {"max square size", generateRandomMsgOnlyData(maxShareCount), false, ""}, + {"1x1(min)", 1}, + {"32x32(med)", 32}, + {"128x128(max)", MaxSquareSize}, } - ctx := context.Background() for _, tc := range testCases { tc := tc - - block := &types.Block{Data: tc.blockData} - t.Run(tc.name, func(t *testing.T) { - err := PutBlock(ctx, dag, block, croute, logger) - if tc.expectErr { - require.Error(t, err) - require.Contains(t, err.Error(), tc.errString) - return - } - + eds, err := PutData(ctx, RandNamespacedShares(t, tc.squareSize*tc.squareSize), dag) require.NoError(t, err) - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) + ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() - block.Hash() - for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() { + for _, rowRoot := range MakeDataHeader(eds).RowsRoots.Bytes() { // recreate the cids using only the computed roots cid, err := plugin.CidFromNamespacedSha256(rowRoot) - if err != nil { - t.Error(err) - } + require.NoError(t, err) // retrieve the data from IPFS - _, err = dag.Get(timeoutCtx, cid) - if err != nil { - t.Errorf("Root not found: %s", cid.String()) - } + _, err = dag.Get(ctx, cid) + require.NoError(t, err) } }) } } -type preprocessingApp struct { - abci.BaseApplication -} - -func (app *preprocessingApp) PreprocessTxs( - req abci.RequestPreprocessTxs) abci.ResponsePreprocessTxs { - time.Sleep(time.Second * 2) - randTxs := generateRandTxs(64, 256) - randMsgs := generateRandNamespacedRawData(128, nmt.DefaultNamespaceIDLen, 256) - randMessages := toMessageSlice(randMsgs) - return abci.ResponsePreprocessTxs{ - Txs: append(req.Txs, randTxs...), - Messages: &tmproto.Messages{MessagesList: randMessages}, - } -} -func generateRandTxs(num int, size int) [][]byte { - randMsgs := generateRandNamespacedRawData(num, nmt.DefaultNamespaceIDLen, size) - for _, msg := range randMsgs { - copy(msg[:nmt.DefaultNamespaceIDLen], consts.TxNamespaceID) - } - return randMsgs -} - -func toMessageSlice(msgs [][]byte) []*tmproto.Message { - res := make([]*tmproto.Message, len(msgs)) - for i := 0; i < len(msgs); i++ { - res[i] = &tmproto.Message{NamespaceId: msgs[i][:nmt.DefaultNamespaceIDLen], Data: msgs[i][nmt.DefaultNamespaceIDLen:]} - } - return res -} - -func TestDataAvailabilityHeaderRewriteBug(t *testing.T) { - logger := log.TestingLogger() - dag := mdutils.Mock() - croute := ipfs.MockRouting() - - txs := types.Txs{} - l := len(txs) - bzs := make([][]byte, l) - for i := 0; i < l; i++ { - bzs[i] = txs[i] - } - app := &preprocessingApp{} - - // See state.CreateProposalBlock to understand why we do this here: - processedBlockTxs := app.PreprocessTxs(abci.RequestPreprocessTxs{Txs: bzs}) - ppt := processedBlockTxs.GetTxs() - - pbmessages := processedBlockTxs.GetMessages() - - lp := len(ppt) - processedTxs := make(types.Txs, lp) - if lp > 0 { - for i := 0; i < l; i++ { - processedTxs[i] = ppt[i] - } - } - - messages := types.MessagesFromProto(pbmessages) - lastID := makeBlockIDRandom() - h := int64(3) - - voteSet, _, vals := randVoteSet(h-1, 1, tmproto.PrecommitType, 10, 1) - commit, err := types.MakeCommit(lastID, h-1, 1, voteSet, vals, time.Now()) - assert.NoError(t, err) - block := types.MakeBlock(1, processedTxs, nil, nil, messages, commit) - block.Hash() - - hash1 := block.DataAvailabilityHeader.Hash() - - ctx := context.TODO() - err = PutBlock(ctx, dag, block, croute, logger) - if err != nil { - t.Fatal(err) - } - - block.Hash() - hash2 := block.DataAvailabilityHeader.Hash() - assert.Equal(t, hash1, hash2) - -} - -func generateRandomMsgOnlyData(msgCount int) types.Data { - out := make([]types.Message, msgCount) - for i, msg := range generateRandNamespacedRawData(msgCount, consts.NamespaceSize, consts.MsgShareSize-2) { - out[i] = types.Message{NamespaceID: msg[:consts.NamespaceSize], Data: msg[consts.NamespaceSize:]} - } - return types.Data{ - Messages: types.Messages{MessagesList: out}, - } -} - -func makeBlockIDRandom() types.BlockID { - var ( - blockHash = make([]byte, tmhash.Size) - partSetHash = make([]byte, tmhash.Size) - ) - mrand.Read(blockHash) - mrand.Read(partSetHash) - return types.BlockID{ - Hash: blockHash, - PartSetHeader: types.PartSetHeader{ - Total: 123, - Hash: partSetHash, - }, - } -} - -func randVoteSet( - height int64, - round int32, - signedMsgType tmproto.SignedMsgType, - numValidators int, - votingPower int64, -) (*types.VoteSet, *types.ValidatorSet, []types.PrivValidator) { - valSet, privValidators := types.RandValidatorSet(numValidators, votingPower) - return types.NewVoteSet("test_chain_id", height, round, signedMsgType, valSet), valSet, privValidators -} +// TODO(Wondertan): Find a proper place for this test +// type preprocessingApp struct { +// abci.BaseApplication +// } +// +// func (app *preprocessingApp) PreprocessTxs( +// req abci.RequestPreprocessTxs) abci.ResponsePreprocessTxs { +// time.Sleep(time.Second * 2) +// randTxs := generateRandTxs(64, 256) +// randMsgs := generateRandNamespacedRawData(128, nmt.DefaultNamespaceIDLen, 256) +// randMessages := toMessageSlice(randMsgs) +// return abci.ResponsePreprocessTxs{ +// Txs: append(req.Txs, randTxs...), +// Messages: &tmproto.Messages{MessagesList: randMessages}, +// } +// } +// func generateRandTxs(num int, size int) [][]byte { +// randMsgs := generateRandNamespacedRawData(num, nmt.DefaultNamespaceIDLen, size) +// for _, msg := range randMsgs { +// copy(msg[:nmt.DefaultNamespaceIDLen], consts.TxNamespaceID) +// } +// return randMsgs +// } +// +// func toMessageSlice(msgs [][]byte) []*tmproto.Message { +// res := make([]*tmproto.Message, len(msgs)) +// for i := 0; i < len(msgs); i++ { +// res[i] = &tmproto.Message{ +// NamespaceId: msgs[i][:nmt.DefaultNamespaceIDLen], +// Data: msgs[i][nmt.DefaultNamespaceIDLen:], +// } +// } +// return res +// } +// +// func TestDataAvailabilityHeaderRewriteBug(t *testing.T) { +// ctx := context.Background() +// dag := mdutils.Mock() +// +// txs := types.Txs{} +// l := len(txs) +// bzs := make([][]byte, l) +// for i := 0; i < l; i++ { +// bzs[i] = txs[i] +// } +// app := &preprocessingApp{} +// +// // See state.CreateProposalBlock to understand why we do this here: +// processedBlockTxs := app.PreprocessTxs(abci.RequestPreprocessTxs{Txs: bzs}) +// ppt := processedBlockTxs.GetTxs() +// +// pbmessages := processedBlockTxs.GetMessages() +// +// lp := len(ppt) +// processedTxs := make(types.Txs, lp) +// if lp > 0 { +// for i := 0; i < l; i++ { +// processedTxs[i] = ppt[i] +// } +// } +// +// messages := types.MessagesFromProto(pbmessages) +// lastID := makeBlockIDRandom() +// h := int64(3) +// +// voteSet, _, vals := randVoteSet(h-1, 1, tmproto.PrecommitType, 10, 1) +// commit, err := types.MakeCommit(lastID, h-1, 1, voteSet, vals, time.Now()) +// assert.NoError(t, err) +// block := types.MakeBlock(1, processedTxs, nil, nil, messages, commit) +// block.Hash() +// +// hash1 := block.DataAvailabilityHeader.Hash() +// +// _, dah, _, err := PutData(ctx, block, dag) +// if err != nil { +// t.Fatal(err) +// } +// +// block.Hash() +// hash2 := block.DataAvailabilityHeader.Hash() +// assert.Equal(t, hash1, hash2) +// } +// +// func makeBlockIDRandom() types.BlockID { +// var ( +// blockHash = make([]byte, tmhash.Size) +// partSetHash = make([]byte, tmhash.Size) +// ) +// mrand.Read(blockHash) +// mrand.Read(partSetHash) +// return types.BlockID{ +// Hash: blockHash, +// PartSetHeader: types.PartSetHeader{ +// Total: 123, +// Hash: partSetHash, +// }, +// } +// } +// +// func randVoteSet( +// height int64, +// round int32, +// signedMsgType tmproto.SignedMsgType, +// numValidators int, +// votingPower int64, +// ) (*types.VoteSet, *types.ValidatorSet, []types.PrivValidator) { +// valSet, privValidators := types.RandValidatorSet(numValidators, votingPower) +// return types.NewVoteSet("test_chain_id", height, round, signedMsgType, valSet), valSet, privValidators +// } From 83f77b652904a82244c9ad2028202796693ddfea Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:45:17 +0300 Subject: [PATCH 06/11] refactor(types): Extract DAHeader and Shares --- types/block.go | 215 +++++---------------------------------- types/block_meta.go | 13 +-- types/block_test.go | 34 +++---- types/light.go | 3 +- types/proposal.go | 25 +++-- types/proposal_test.go | 11 +- types/share_splitting.go | 28 ++--- types/shares.go | 33 ------ types/shares_test.go | 50 ++++----- types/tx.go | 3 +- 10 files changed, 111 insertions(+), 304 deletions(-) diff --git a/types/block.go b/types/block.go index 3ce4c58c7c..fc8269580a 100644 --- a/types/block.go +++ b/types/block.go @@ -11,7 +11,6 @@ import ( "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" "github.com/lazyledger/nmt/namespace" - "github.com/lazyledger/rsmt2d" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/merkle" @@ -21,7 +20,7 @@ import ( tmmath "github.com/lazyledger/lazyledger-core/libs/math" "github.com/lazyledger/lazyledger-core/libs/protoio" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" - "github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" tmversion "github.com/lazyledger/lazyledger-core/proto/tendermint/version" "github.com/lazyledger/lazyledger-core/types/consts" @@ -45,121 +44,14 @@ const ( MaxOverheadForBlock int64 = 11 ) -// DataAvailabilityHeader (DAHeader) contains the row and column roots of the erasure -// coded version of the data in Block.Data. -// Therefor the original Block.Data is arranged in a -// k × k matrix, which is then "extended" to a -// 2k × 2k matrix applying multiple times Reed-Solomon encoding. -// For details see Section 5.2: https://arxiv.org/abs/1809.09044 -// or the LazyLedger specification: -// https://github.com/lazyledger/lazyledger-specs/blob/master/specs/data_structures.md#availabledataheader -// Note that currently we list row and column roots in separate fields -// (different from the spec). -type DataAvailabilityHeader struct { - // RowRoot_j = root((M_{j,1} || M_{j,2} || ... || M_{j,2k} )) - RowsRoots NmtRoots `json:"row_roots"` - // ColumnRoot_j = root((M_{1,j} || M_{2,j} || ... || M_{2k,j} )) - ColumnRoots NmtRoots `json:"column_roots"` - // cached result of Hash() not to be recomputed - hash []byte -} - -type NmtRoots []namespace.IntervalDigest - -func (roots NmtRoots) Bytes() [][]byte { - res := make([][]byte, len(roots)) - for i := 0; i < len(roots); i++ { - res[i] = roots[i].Bytes() - } - return res -} - -func NmtRootsFromBytes(in [][]byte) (roots NmtRoots, err error) { - roots = make([]namespace.IntervalDigest, len(in)) - for i := 0; i < len(in); i++ { - roots[i], err = namespace.IntervalDigestFromBytes(consts.NamespaceSize, in[i]) - if err != nil { - return roots, err - } - } - return -} - -// String returns hex representation of merkle hash of the DAHeader. -func (dah *DataAvailabilityHeader) String() string { - if dah == nil { - return "" - } - return fmt.Sprintf("%X", dah.Hash()) -} - -// Equals checks equality of two DAHeaders. -func (dah *DataAvailabilityHeader) Equals(to *DataAvailabilityHeader) bool { - return bytes.Equal(dah.Hash(), to.Hash()) -} - -// Hash computes and caches the merkle root of the row and column roots. -func (dah *DataAvailabilityHeader) Hash() []byte { - if dah == nil { - return merkle.HashFromByteSlices(nil) - } - if len(dah.hash) != 0 { - return dah.hash - } - - colsCount := len(dah.ColumnRoots) - rowsCount := len(dah.RowsRoots) - slices := make([][]byte, colsCount+rowsCount) - for i, rowRoot := range dah.RowsRoots { - slices[i] = rowRoot.Bytes() - } - for i, colRoot := range dah.ColumnRoots { - slices[i+colsCount] = colRoot.Bytes() - } - // The single data root is computed using a simple binary merkle tree. - // Effectively being root(rowRoots || columnRoots): - dah.hash = merkle.HashFromByteSlices(slices) - return dah.hash -} - -func (dah *DataAvailabilityHeader) ToProto() (*tmproto.DataAvailabilityHeader, error) { - if dah == nil { - return nil, errors.New("nil DataAvailabilityHeader") - } - - dahp := new(tmproto.DataAvailabilityHeader) - dahp.RowRoots = dah.RowsRoots.Bytes() - dahp.ColumnRoots = dah.ColumnRoots.Bytes() - return dahp, nil -} - -func DataAvailabilityHeaderFromProto(dahp *tmproto.DataAvailabilityHeader) (dah *DataAvailabilityHeader, err error) { - if dahp == nil { - return nil, errors.New("nil DataAvailabilityHeader") - } - - dah = new(DataAvailabilityHeader) - dah.RowsRoots, err = NmtRootsFromBytes(dahp.RowRoots) - if err != nil { - return - } - - dah.ColumnRoots, err = NmtRootsFromBytes(dahp.ColumnRoots) - if err != nil { - return - } - - return -} - // Block defines the atomic unit of a Tendermint blockchain. type Block struct { mtx tmsync.Mutex Header `json:"header"` Data `json:"data"` - DataAvailabilityHeader DataAvailabilityHeader `json:"availability_header"` - LastCommit *Commit `json:"last_commit"` + DataAvailabilityHeader ipld.DataAvailabilityHeader `json:"availability_header"` + LastCommit *Commit `json:"last_commit"` } // ValidateBasic performs basic validation that doesn't involve state data. @@ -189,11 +81,6 @@ func (b *Block) ValidateBasic() error { return fmt.Errorf("wrong Header.LastCommitHash. Expected %X, got %X", w, g) } - // NOTE: b.Data.Txs may be nil, but b.Data.Hash() still works fine. - if w, g := b.DataAvailabilityHeader.Hash(), b.DataHash; !bytes.Equal(w, g) { - return fmt.Errorf("wrong Header.DataHash. Expected %X, got %X", w, g) - } - // NOTE: b.Evidence.Evidence may be nil, but we're just looping. for i, ev := range b.Evidence.Evidence { if err := ev.ValidateBasic(); err != nil { @@ -208,66 +95,6 @@ func (b *Block) ValidateBasic() error { return nil } -// fillHeader fills in any remaining header fields that are a function of the block data -func (b *Block) fillHeader() { - if b.LastCommitHash == nil { - b.LastCommitHash = b.LastCommit.Hash() - } - if b.DataHash == nil || b.DataAvailabilityHeader.hash == nil { - b.fillDataAvailabilityHeader() - } - if b.EvidenceHash == nil { - b.EvidenceHash = b.Evidence.Hash() - } -} - -// TODO: Move out from 'types' package -// fillDataAvailabilityHeader fills in any remaining DataAvailabilityHeader fields -// that are a function of the block data. -func (b *Block) fillDataAvailabilityHeader() { - namespacedShares, dataSharesLen := b.Data.ComputeShares() - shares := namespacedShares.RawShares() - - // create the nmt wrapper to generate row and col commitments - squareSize := uint32(math.Sqrt(float64(len(shares)))) - tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize)) - - // TODO(ismail): for better efficiency and a larger number shares - // we should switch to the rsmt2d.LeopardFF16 codec: - extendedDataSquare, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), tree.Constructor) - if err != nil { - panic(fmt.Sprintf("unexpected error: %v", err)) - } - - // generate the row and col roots using the EDS and nmt wrapper - rowRoots := extendedDataSquare.RowRoots() - colRoots := extendedDataSquare.ColumnRoots() - - b.DataAvailabilityHeader = DataAvailabilityHeader{ - RowsRoots: make([]namespace.IntervalDigest, extendedDataSquare.Width()), - ColumnRoots: make([]namespace.IntervalDigest, extendedDataSquare.Width()), - } - - // todo(evan): remove interval digests - // convert the roots to interval digests - for i := 0; i < len(rowRoots); i++ { - rowRoot, err := namespace.IntervalDigestFromBytes(consts.NamespaceSize, rowRoots[i]) - if err != nil { - panic(err) - } - colRoot, err := namespace.IntervalDigestFromBytes(consts.NamespaceSize, colRoots[i]) - if err != nil { - panic(err) - } - b.DataAvailabilityHeader.RowsRoots[i] = rowRoot - b.DataAvailabilityHeader.ColumnRoots[i] = colRoot - } - - // return the root hash of DA Header - b.DataHash = b.DataAvailabilityHeader.Hash() - b.NumOriginalDataShares = uint64(dataSharesLen) -} - // Hash computes and returns the block hash. // If the block is incomplete, block hash is nil for safety. func (b *Block) Hash() tmbytes.HexBytes { @@ -276,11 +103,17 @@ func (b *Block) Hash() tmbytes.HexBytes { } b.mtx.Lock() defer b.mtx.Unlock() - - if b.LastCommit == nil { + // short circuit if block is incomplete + if b.LastCommit == nil || b.DataHash == nil { return nil } - b.fillHeader() + // fill remaining fields + if b.Header.LastCommitHash == nil { + b.Header.LastCommitHash = b.LastCommit.Hash() + } + if b.Header.EvidenceHash == nil { + b.Header.EvidenceHash = b.Evidence.Hash() + } return b.Header.Hash() } @@ -413,7 +246,7 @@ func BlockFromProto(bp *tmproto.Block) (*Block, error) { return nil, err } - dah, err := DataAvailabilityHeaderFromProto(bp.DataAvailabilityHeader) + dah, err := ipld.DataAvailabilityHeaderFromProto(bp.DataAvailabilityHeader) if err != nil { return nil, err } @@ -479,22 +312,24 @@ func MaxDataBytesNoEvidence(maxBytes int64, valsCount int) int64 { // It populates the same set of fields validated by ValidateBasic. func MakeBlock( height int64, - txs []Tx, evidence []Evidence, intermediateStateRoots []tmbytes.HexBytes, messages Messages, + txs []Tx, evidences []Evidence, intermediateStateRoots []tmbytes.HexBytes, messages Messages, lastCommit *Commit) *Block { + evidence := EvidenceData{Evidence: evidences} block := &Block{ Header: Header{ - Version: tmversion.Consensus{Block: version.BlockProtocol, App: 0}, - Height: height, + Version: tmversion.Consensus{Block: version.BlockProtocol, App: 0}, + Height: height, + LastCommitHash: lastCommit.Hash(), + EvidenceHash: evidence.Hash(), }, Data: Data{ Txs: txs, IntermediateStateRoots: IntermediateStateRoots{RawRootsList: intermediateStateRoots}, - Evidence: EvidenceData{Evidence: evidence}, + Evidence: evidence, Messages: messages, }, LastCommit: lastCommit, } - block.fillHeader() return block } @@ -1219,7 +1054,7 @@ type IntermediateStateRoots struct { RawRootsList []tmbytes.HexBytes `json:"intermediate_roots"` } -func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares { +func (roots IntermediateStateRoots) splitIntoShares() ipld.NamespacedShares { rawDatas := make([][]byte, 0, len(roots.RawRootsList)) for _, root := range roots.RawRootsList { rawData, err := root.MarshalDelimited() @@ -1232,8 +1067,8 @@ func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares { return shares } -func (msgs Messages) splitIntoShares() NamespacedShares { - shares := make([]NamespacedShare, 0) +func (msgs Messages) splitIntoShares() ipld.NamespacedShares { + shares := make([]ipld.NamespacedShare, 0) for _, m := range msgs.MessagesList { rawData, err := m.MarshalDelimited() if err != nil { @@ -1246,7 +1081,7 @@ func (msgs Messages) splitIntoShares() NamespacedShares { // ComputeShares splits block data into shares of an original data square and // returns them along with an amount of non-redundant shares. -func (data *Data) ComputeShares() (NamespacedShares, int) { +func (data *Data) ComputeShares() (ipld.NamespacedShares, int) { // TODO(ismail): splitting into shares should depend on the block size and layout // see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/block_proposer.md#laying-out-transactions-and-messages @@ -1525,7 +1360,7 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceList) error { return nil } -func (data *EvidenceData) splitIntoShares() NamespacedShares { +func (data *EvidenceData) splitIntoShares() ipld.NamespacedShares { rawDatas := make([][]byte, 0, len(data.Evidence)) for _, ev := range data.Evidence { pev, err := EvidenceToProto(ev) diff --git a/types/block_meta.go b/types/block_meta.go index 0926fa91a9..371d228fbf 100644 --- a/types/block_meta.go +++ b/types/block_meta.go @@ -5,16 +5,17 @@ import ( "errors" "fmt" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" ) // BlockMeta contains meta information. type BlockMeta struct { - BlockID BlockID `json:"block_id"` - BlockSize int `json:"block_size"` - Header Header `json:"header"` - NumTxs int `json:"num_txs"` - DAHeader DataAvailabilityHeader `json:"da_header"` + BlockID BlockID `json:"block_id"` + BlockSize int `json:"block_size"` + Header Header `json:"header"` + NumTxs int `json:"num_txs"` + DAHeader ipld.DataAvailabilityHeader `json:"da_header"` } // NewBlockMeta returns a new BlockMeta. @@ -65,7 +66,7 @@ func BlockMetaFromProto(pb *tmproto.BlockMeta) (*BlockMeta, error) { return nil, err } - dah, err := DataAvailabilityHeaderFromProto(pb.DaHeader) + dah, err := ipld.DataAvailabilityHeaderFromProto(pb.DaHeader) if err != nil { return nil, err } diff --git a/types/block_test.go b/types/block_test.go index 5a0ea73e85..8238472aac 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -4,6 +4,7 @@ import ( // it is ok to use math/rand here: we do not need a cryptographically secure random // number generator here and we can run the tests a bit faster stdbytes "bytes" + "context" "encoding/hex" "math" mrand "math/rand" @@ -14,6 +15,7 @@ import ( "time" gogotypes "github.com/gogo/protobuf/types" + mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -23,6 +25,7 @@ import ( "github.com/lazyledger/lazyledger-core/libs/bits" "github.com/lazyledger/lazyledger-core/libs/bytes" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" tmversion "github.com/lazyledger/lazyledger-core/proto/tendermint/version" "github.com/lazyledger/lazyledger-core/types/consts" @@ -80,13 +83,6 @@ func TestBlockValidateBasic(t *testing.T) { blk.LastCommit.hash = nil // clear hash or change wont be noticed }, true}, {"Remove LastCommitHash", func(blk *Block) { blk.LastCommitHash = []byte("something else") }, true}, - {"Tampered Data", func(blk *Block) { - blk.Data.Txs[0] = Tx("something else") - blk.DataHash = nil // clear hash or change wont be noticed - }, true}, - {"Tampered DataHash", func(blk *Block) { - blk.DataHash = tmrand.Bytes(len(blk.DataHash)) - }, true}, {"Tampered EvidenceHash", func(blk *Block) { blk.EvidenceHash = tmrand.Bytes(len(blk.EvidenceHash)) }, true}, @@ -109,6 +105,8 @@ func TestBlockValidateBasic(t *testing.T) { i := i t.Run(tc.testName, func(t *testing.T) { block := MakeBlock(h, txs, evList, nil, Messages{}, commit) + _, err = block.RowSet(context.TODO(), mdutils.Mock()) + require.NoError(t, err) block.ProposerAddress = valSet.GetProposer().Address tc.malleateBlock(block) err = block.ValidateBasic() @@ -146,7 +144,7 @@ func TestBlockMakePartSetWithEvidence(t *testing.T) { partSet := MakeBlock(h, []Tx{Tx("Hello World")}, evList, nil, Messages{}, commit).MakePartSet(512) assert.NotNil(t, partSet) - assert.EqualValues(t, 5, partSet.Total()) + assert.EqualValues(t, 4, partSet.Total()) } func TestBlockHashesTo(t *testing.T) { @@ -163,6 +161,8 @@ func TestBlockHashesTo(t *testing.T) { block := MakeBlock(h, []Tx{Tx("Hello World")}, evList, nil, Messages{}, commit) block.ValidatorsHash = valSet.Hash() + _, err = block.RowSet(context.TODO(), mdutils.Mock()) + require.NoError(t, err) assert.False(t, block.HashesTo([]byte{})) assert.False(t, block.HashesTo([]byte("something else"))) assert.True(t, block.HashesTo(block.Hash())) @@ -212,10 +212,10 @@ func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) BlockID { } } -func makeDAHeaderRandom() *DataAvailabilityHeader { - rows, _ := NmtRootsFromBytes([][]byte{tmrand.Bytes(2*consts.NamespaceSize + tmhash.Size)}) - clns, _ := NmtRootsFromBytes([][]byte{tmrand.Bytes(2*consts.NamespaceSize + tmhash.Size)}) - return &DataAvailabilityHeader{ +func makeDAHeaderRandom() *ipld.DataAvailabilityHeader { + rows, _ := ipld.NmtRootsFromBytes([][]byte{tmrand.Bytes(2*consts.NamespaceSize + tmhash.Size)}) + clns, _ := ipld.NmtRootsFromBytes([][]byte{tmrand.Bytes(2*consts.NamespaceSize + tmhash.Size)}) + return &ipld.DataAvailabilityHeader{ RowsRoots: rows, ColumnRoots: clns, } @@ -223,21 +223,11 @@ func makeDAHeaderRandom() *DataAvailabilityHeader { var nilBytes []byte -// This follows RFC-6962, i.e. `echo -n '' | sha256sum` -var emptyBytes = []byte{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, - 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, - 0x78, 0x52, 0xb8, 0x55} - func TestNilHeaderHashDoesntCrash(t *testing.T) { assert.Equal(t, nilBytes, []byte((*Header)(nil).Hash())) assert.Equal(t, nilBytes, []byte((new(Header)).Hash())) } -func TestNilDataAvailabilityHeaderHashDoesntCrash(t *testing.T) { - assert.Equal(t, emptyBytes, (*DataAvailabilityHeader)(nil).Hash()) - assert.Equal(t, emptyBytes, new(DataAvailabilityHeader).Hash()) -} - func TestEmptyBlockData(t *testing.T) { blockData := Data{} shares, _ := blockData.ComputeShares() diff --git a/types/light.go b/types/light.go index 7c7bff681f..b696b46e32 100644 --- a/types/light.go +++ b/types/light.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" ) @@ -16,7 +17,7 @@ type LightBlock struct { ValidatorSet *ValidatorSet `json:"validator_set"` // DataAvailabilityHeader is only populated for DAS light clients for others it can be nil. - DataAvailabilityHeader *DataAvailabilityHeader `json:"data_availability_header"` + DataAvailabilityHeader *ipld.DataAvailabilityHeader `json:"data_availability_header"` } // ValidateBasic checks that the data is correct and consistent diff --git a/types/proposal.go b/types/proposal.go index 32740b4363..2cebb25afe 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -7,6 +7,7 @@ import ( tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes" "github.com/lazyledger/lazyledger-core/libs/protoio" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" tmtime "github.com/lazyledger/lazyledger-core/types/time" ) @@ -24,18 +25,24 @@ var ( // If POLRound >= 0, then BlockID corresponds to the block that is locked in POLRound. type Proposal struct { Type tmproto.SignedMsgType - Height int64 `json:"height"` - Round int32 `json:"round"` // there can not be greater than 2_147_483_647 rounds - POLRound int32 `json:"pol_round"` // -1 if null. - BlockID BlockID `json:"block_id"` - Timestamp time.Time `json:"timestamp"` - Signature []byte `json:"signature"` - DAHeader *DataAvailabilityHeader `json:"da_header"` + Height int64 `json:"height"` + Round int32 `json:"round"` // there can not be greater than 2_147_483_647 rounds + POLRound int32 `json:"pol_round"` // -1 if null. + BlockID BlockID `json:"block_id"` + Timestamp time.Time `json:"timestamp"` + Signature []byte `json:"signature"` + DAHeader *ipld.DataAvailabilityHeader `json:"da_header"` } // NewProposal returns a new Proposal. // If there is no POLRound, polRound should be -1. -func NewProposal(height int64, round int32, polRound int32, blockID BlockID, daH *DataAvailabilityHeader) *Proposal { +func NewProposal( + height int64, + round int32, + polRound int32, + blockID BlockID, + daH *ipld.DataAvailabilityHeader, +) *Proposal { return &Proposal{ Type: tmproto.ProposalType, Height: height, @@ -157,7 +164,7 @@ func ProposalFromProto(pp *tmproto.Proposal) (*Proposal, error) { return nil, err } - dah, err := DataAvailabilityHeaderFromProto(pp.DAHeader) + dah, err := ipld.DataAvailabilityHeaderFromProto(pp.DAHeader) if err != nil { return nil, err } diff --git a/types/proposal_test.go b/types/proposal_test.go index e4939775bc..dbec4c3d26 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -12,6 +12,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto/tmhash" "github.com/lazyledger/lazyledger-core/libs/protoio" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" ) @@ -21,8 +22,8 @@ var ( ) func init() { - rows, _ := NmtRootsFromBytes([][]byte{[]byte("HeHasBeenElected--June_15_2020_amino_was_removed")}) - clmns, _ := NmtRootsFromBytes([][]byte{[]byte("HeHasBeenElected--June_15_2020_amino_was_removed")}) + rows, _ := ipld.NmtRootsFromBytes([][]byte{[]byte("HeHasBeenElected--June_15_2020_amino_was_removed")}) + clmns, _ := ipld.NmtRootsFromBytes([][]byte{[]byte("HeHasBeenElected--June_15_2020_amino_was_removed")}) var stamp, err = time.Parse(TimeFormat, "2018-02-11T07:09:22.765Z") if err != nil { @@ -35,7 +36,7 @@ func init() { PartSetHeader: PartSetHeader{Total: 111, Hash: []byte("--June_15_2020_amino_was_removed")}}, POLRound: -1, Timestamp: stamp, - DAHeader: &DataAvailabilityHeader{ + DAHeader: &ipld.DataAvailabilityHeader{ RowsRoots: rows, ColumnRoots: clmns, }, @@ -185,7 +186,7 @@ func TestProposalProtoBuf(t *testing.T) { makeDAHeaderRandom(), ) proposal.Signature = []byte("sig") - proposal2 := NewProposal(1, 2, 3, BlockID{}, &DataAvailabilityHeader{}) + proposal2 := NewProposal(1, 2, 3, BlockID{}, &ipld.DataAvailabilityHeader{}) testCases := []struct { msg string @@ -194,7 +195,7 @@ func TestProposalProtoBuf(t *testing.T) { }{ {"success", proposal, true}, {"success", proposal2, false}, // blockID cannot be empty - {"empty proposal failure validatebasic", &Proposal{DAHeader: &DataAvailabilityHeader{}}, false}, + {"empty proposal failure validatebasic", &Proposal{DAHeader: &ipld.DataAvailabilityHeader{}}, false}, {"nil proposal", nil, false}, } for _, tc := range testCases { diff --git a/types/share_splitting.go b/types/share_splitting.go index bf55629ac3..33a10f65e2 100644 --- a/types/share_splitting.go +++ b/types/share_splitting.go @@ -3,13 +3,15 @@ package types import ( "bytes" - "github.com/lazyledger/lazyledger-core/types/consts" "github.com/lazyledger/nmt/namespace" + + "github.com/lazyledger/lazyledger-core/p2p/ipld" + "github.com/lazyledger/lazyledger-core/types/consts" ) // appendToShares appends raw data as shares. // Used for messages. -func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare { +func appendToShares(shares []ipld.NamespacedShare, nid namespace.ID, rawData []byte) []ipld.NamespacedShare { if len(rawData) <= consts.MsgShareSize { rawShare := append(append( make([]byte, 0, len(nid)+len(rawData)), @@ -17,7 +19,7 @@ func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) rawData..., ) paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize) - share := NamespacedShare{paddedShare, nid} + share := ipld.NamespacedShare{Share: paddedShare, ID: nid} shares = append(shares, share) } else { // len(rawData) > MsgShareSize shares = append(shares, splitMessage(rawData, nid)...) @@ -27,14 +29,14 @@ func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) // splitMessage breaks the data in a message into the minimum number of // namespaced shares -func splitMessage(rawData []byte, nid namespace.ID) []NamespacedShare { - shares := make([]NamespacedShare, 0) +func splitMessage(rawData []byte, nid namespace.ID) []ipld.NamespacedShare { + shares := make([]ipld.NamespacedShare, 0) firstRawShare := append(append( make([]byte, 0, consts.ShareSize), nid...), rawData[:consts.MsgShareSize]..., ) - shares = append(shares, NamespacedShare{firstRawShare, nid}) + shares = append(shares, ipld.NamespacedShare{Share: firstRawShare, ID: nid}) rawData = rawData[consts.MsgShareSize:] for len(rawData) > 0 { shareSizeOrLen := min(consts.MsgShareSize, len(rawData)) @@ -44,7 +46,7 @@ func splitMessage(rawData []byte, nid namespace.ID) []NamespacedShare { rawData[:shareSizeOrLen]..., ) paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize) - share := NamespacedShare{paddedShare, nid} + share := ipld.NamespacedShare{Share: paddedShare, ID: nid} shares = append(shares, share) rawData = rawData[shareSizeOrLen:] } @@ -53,8 +55,8 @@ func splitMessage(rawData []byte, nid namespace.ID) []NamespacedShare { // splitContiguous splits multiple raw data contiguously as shares. // Used for transactions, intermediate state roots, and evidence. -func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare { - shares := make([]NamespacedShare, 0) +func splitContiguous(nid namespace.ID, rawDatas [][]byte) []ipld.NamespacedShare { + shares := make([]ipld.NamespacedShare, 0) // Index into the outer slice of rawDatas outerIndex := 0 // Index into the inner slice of rawDatas @@ -69,7 +71,7 @@ func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare { byte(startIndex)), rawData...) paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize) - share := NamespacedShare{paddedShare, nid} + share := ipld.NamespacedShare{Share: paddedShare, ID: nid} shares = append(shares, share) } return shares @@ -117,10 +119,10 @@ var tailPaddingShare = append( bytes.Repeat([]byte{0}, consts.ShareSize-consts.NamespaceSize)..., ) -func TailPaddingShares(n int) NamespacedShares { - shares := make([]NamespacedShare, n) +func TailPaddingShares(n int) ipld.NamespacedShares { + shares := make([]ipld.NamespacedShare, n) for i := 0; i < n; i++ { - shares[i] = NamespacedShare{ + shares[i] = ipld.NamespacedShare{ Share: tailPaddingShare, ID: consts.TailPaddingNamespaceID, } diff --git a/types/shares.go b/types/shares.go index b96a83e96a..0f75b58116 100644 --- a/types/shares.go +++ b/types/shares.go @@ -2,41 +2,8 @@ package types import ( "encoding/binary" - - "github.com/lazyledger/nmt/namespace" ) -// Share contains the raw share data without the corresponding namespace. -type Share []byte - -// NamespacedShare extends a Share with the corresponding namespace. -type NamespacedShare struct { - Share - ID namespace.ID -} - -func (n NamespacedShare) NamespaceID() namespace.ID { - return n.ID -} - -func (n NamespacedShare) Data() []byte { - return n.Share -} - -// NamespacedShares is just a list of NamespacedShare elements. -// It can be used to extract the raw raw shares. -type NamespacedShares []NamespacedShare - -// RawShares returns the raw shares that can be fed into the erasure coding -// library (e.g. rsmt2d). -func (ns NamespacedShares) RawShares() [][]byte { - res := make([][]byte, len(ns)) - for i, nsh := range ns { - res[i] = nsh.Share - } - return res -} - func (tx Tx) MarshalDelimited() ([]byte, error) { lenBuf := make([]byte, binary.MaxVarintLen64) length := uint64(len(tx)) diff --git a/types/shares_test.go b/types/shares_test.go index 8bcd23f842..1152027622 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -10,16 +10,18 @@ import ( "testing" "time" - tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes" - "github.com/lazyledger/lazyledger-core/libs/protoio" - "github.com/lazyledger/lazyledger-core/types/consts" "github.com/lazyledger/nmt/namespace" "github.com/lazyledger/rsmt2d" "github.com/stretchr/testify/assert" + + tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes" + "github.com/lazyledger/lazyledger-core/libs/protoio" + "github.com/lazyledger/lazyledger-core/p2p/ipld" + "github.com/lazyledger/lazyledger-core/types/consts" ) type splitter interface { - splitIntoShares() NamespacedShares + splitIntoShares() ipld.NamespacedShares } func TestMakeShares(t *testing.T) { @@ -58,20 +60,20 @@ func TestMakeShares(t *testing.T) { tests := []struct { name string args args - want NamespacedShares + want ipld.NamespacedShares }{ {"evidence", args{ data: &EvidenceData{ Evidence: []Evidence{testEvidence}, }, - }, NamespacedShares{NamespacedShare{ + }, ipld.NamespacedShares{ipld.NamespacedShare{ Share: append( append(reservedEvidenceNamespaceID, byte(0)), testEvidenceBytes[:consts.TxShareSize]..., ), ID: reservedEvidenceNamespaceID, - }, NamespacedShare{ + }, ipld.NamespacedShare{ Share: append( append(reservedEvidenceNamespaceID, byte(0)), zeroPadIfNecessary(testEvidenceBytes[consts.TxShareSize:], consts.TxShareSize)..., @@ -83,8 +85,8 @@ func TestMakeShares(t *testing.T) { args{ data: Txs{smolTx}, }, - NamespacedShares{ - NamespacedShare{ + ipld.NamespacedShares{ + ipld.NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(smolTxLenDelimited, consts.TxShareSize)..., @@ -97,15 +99,15 @@ func TestMakeShares(t *testing.T) { args{ data: Txs{largeTx}, }, - NamespacedShares{ - NamespacedShare{ + ipld.NamespacedShares{ + ipld.NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), largeTxLenDelimited[:consts.TxShareSize]..., ), ID: reservedTxNamespaceID, }, - NamespacedShare{ + ipld.NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(largeTxLenDelimited[consts.TxShareSize:], consts.TxShareSize)..., @@ -118,15 +120,15 @@ func TestMakeShares(t *testing.T) { args{ data: Txs{largeTx, smolTx}, }, - NamespacedShares{ - NamespacedShare{ + ipld.NamespacedShares{ + ipld.NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), largeTxLenDelimited[:consts.TxShareSize]..., ), ID: reservedTxNamespaceID, }, - NamespacedShare{ + ipld.NamespacedShare{ Share: append( append( reservedTxNamespaceID, @@ -145,8 +147,8 @@ func TestMakeShares(t *testing.T) { args{ data: Messages{[]Message{msg1}}, }, - NamespacedShares{ - NamespacedShare{ + ipld.NamespacedShares{ + ipld.NamespacedShare{ Share: append( []byte(msg1.NamespaceID), zeroPadIfNecessary(msg1Marshaled, consts.MsgShareSize)..., @@ -193,7 +195,7 @@ func Test_zeroPadIfNecessary(t *testing.T) { } func Test_appendToSharesOverwrite(t *testing.T) { - var shares NamespacedShares + var shares ipld.NamespacedShares // generate some arbitrary namespaced shares first share that must be split newShare := generateRandomNamespacedShares(1, consts.MsgShareSize+1)[0] @@ -245,7 +247,7 @@ func TestDataFromSquare(t *testing.T) { ) shares, _ := data.ComputeShares() - rawShares := shares.RawShares() + rawShares := shares.Raw() eds, err := rsmt2d.ComputeExtendedDataSquare(rawShares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) if err != nil { @@ -322,7 +324,7 @@ func Test_processContiguousShares(t *testing.T) { shares := txs.splitIntoShares() - parsedTxs, err := processContiguousShares(shares.RawShares()) + parsedTxs, err := processContiguousShares(shares.Raw()) if err != nil { t.Error(err) } @@ -339,7 +341,7 @@ func Test_processContiguousShares(t *testing.T) { shares := txs.splitIntoShares() - parsedTxs, err := processContiguousShares(shares.RawShares()) + parsedTxs, err := processContiguousShares(shares.Raw()) if err != nil { t.Error(err) } @@ -403,7 +405,7 @@ func Test_parseMsgShares(t *testing.T) { shares := msgs.splitIntoShares() - parsedMsgs, err := parseMsgShares(shares.RawShares()) + parsedMsgs, err := parseMsgShares(shares.Raw()) if err != nil { t.Error(err) } @@ -420,7 +422,7 @@ func Test_parseMsgShares(t *testing.T) { msgs := generateRandomlySizedMessages(tc.msgCount, tc.msgSize) shares := msgs.splitIntoShares() - parsedMsgs, err := parseMsgShares(shares.RawShares()) + parsedMsgs, err := parseMsgShares(shares.Raw()) if err != nil { t.Error(err) } @@ -525,7 +527,7 @@ func generateRandomMessage(size int) Message { return msg } -func generateRandomNamespacedShares(count, msgSize int) NamespacedShares { +func generateRandomNamespacedShares(count, msgSize int) ipld.NamespacedShares { shares := generateRandNamespacedRawData(count, consts.NamespaceSize, msgSize) msgs := make([]Message, count) for i, s := range shares { diff --git a/types/tx.go b/types/tx.go index bbd4ce18c5..f446436625 100644 --- a/types/tx.go +++ b/types/tx.go @@ -8,6 +8,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto/merkle" "github.com/lazyledger/lazyledger-core/crypto/tmhash" tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" "github.com/lazyledger/lazyledger-core/types/consts" ) @@ -80,7 +81,7 @@ func (txs Txs) Proof(i int) TxProof { } } -func (txs Txs) splitIntoShares() NamespacedShares { +func (txs Txs) splitIntoShares() ipld.NamespacedShares { rawDatas := make([][]byte, len(txs)) for i, tx := range txs { rawData, err := tx.MarshalDelimited() From fe3ec089cd87e4305053b8cfe99c28f181072954 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:49:46 +0300 Subject: [PATCH 07/11] feat(types): introduce RowSet --- types/block.go | 19 +++++++++++++++++++ types/part_set.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/types/block.go b/types/block.go index fc8269580a..021d8aff1f 100644 --- a/types/block.go +++ b/types/block.go @@ -2,6 +2,7 @@ package types import ( "bytes" + "context" "errors" "fmt" "math" @@ -10,6 +11,7 @@ import ( "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" + format "github.com/ipfs/go-ipld-format" "github.com/lazyledger/nmt/namespace" "github.com/lazyledger/lazyledger-core/crypto" @@ -138,6 +140,23 @@ func (b *Block) MakePartSet(partSize uint32) *PartSet { return NewPartSetFromData(bz, partSize) } +// TODO(Wondertan): Aim for Block to be complete without calling RowSet +func (b *Block) RowSet(ctx context.Context, adder format.NodeAdder) (*RowSet, error) { + shares, dataLen := b.ComputeShares() + eds, err := ipld.PutData(ctx, shares, adder) + if err != nil { + return nil, fmt.Errorf("failed to put Block into DAG: %w", err) + } + + // TODO(Wondertan): Finish DAHeader removal from Block + // Currently needed for StoreTests + b.DataAvailabilityHeader = *ipld.MakeDataHeader(eds) + + b.Header.NumOriginalDataShares = uint64(dataLen) + b.Header.DataHash = b.DataAvailabilityHeader.Hash() + return NewRowSet(eds), nil +} + // HashesTo is a convenience function that checks if a block hashes to the given argument. // Returns false if the block is nil or the hash is empty. func (b *Block) HashesTo(hash []byte) bool { diff --git a/types/part_set.go b/types/part_set.go index 59f02cbc97..dc63697f89 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -6,13 +6,17 @@ import ( "fmt" "io" + "github.com/lazyledger/rsmt2d" + "github.com/lazyledger/lazyledger-core/crypto/merkle" "github.com/lazyledger/lazyledger-core/libs/bits" tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes" tmjson "github.com/lazyledger/lazyledger-core/libs/json" tmmath "github.com/lazyledger/lazyledger-core/libs/math" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" + "github.com/lazyledger/lazyledger-core/types/consts" ) var ( @@ -373,3 +377,28 @@ func (ps *PartSet) MarshalJSON() ([]byte, error) { ps.partsBitArray, }) } + +type RowSet struct { + DAHeader *ipld.DataAvailabilityHeader + + rows [][]byte + rowsHave *bits.BitArray +} + +func NewRowSet(eds *rsmt2d.ExtendedDataSquare) *RowSet { + width := int(eds.Width() / 2) + rows := make([][]byte, width) + for i := 0; i < width; i++ { + row := eds.Row(uint(i)) + rows[i] = make([]byte, 0, consts.ShareSize*width) + for j := 0; j < width; j++ { + rows[i] = append(rows[i], row[j]...) + } + } + + return &RowSet{ + DAHeader: ipld.MakeDataHeader(eds), + rows: rows, + rowsHave: bits.NewBitArray(width), + } +} From 3fd37f98f2917bc3d0c94b3342e560473a19e2c9 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:52:34 +0300 Subject: [PATCH 08/11] feat: rely on new RowSet method and new API in p2p/ipld in consensus --- consensus/state.go | 49 ++++++++++++++++++++++------------ consensus/types/round_state.go | 7 +++-- node/node.go | 1 + state/execution.go | 27 ++++++++++++++++--- state/state.go | 4 +-- 5 files changed, 64 insertions(+), 24 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 90112e15a7..c7c443f0fe 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -156,8 +156,8 @@ type State struct { metrics *Metrics // context of the recent proposed block - proposalCtx context.Context - proposalCancel context.CancelFunc + provideCtx context.Context + provideCancel context.CancelFunc } // StateOption sets an optional parameter on the State. @@ -658,12 +658,15 @@ func (cs *State) updateToState(state sm.State) { cs.Proposal = nil cs.ProposalBlock = nil cs.ProposalBlockParts = nil + cs.ProposalBlockRows = nil cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil + cs.LockedBlockRows = nil cs.ValidRound = -1 cs.ValidBlock = nil cs.ValidBlockParts = nil + cs.ValidBlockRows = nil cs.Votes = cstypes.NewHeightVoteSet(state.ChainID, height, validators) cs.CommitRound = -1 cs.LastValidators = state.LastValidators @@ -962,6 +965,7 @@ func (cs *State) enterNewRound(height int64, round int32) { cs.Proposal = nil cs.ProposalBlock = nil cs.ProposalBlockParts = nil + cs.ProposalBlockRows = nil } cs.Votes.SetRound(tmmath.SafeAddInt32(round, 1)) // also track next round (round+1) to allow round-skipping cs.TriggeredTimeoutPrecommit = false @@ -1078,14 +1082,15 @@ func (cs *State) isProposer(address []byte) bool { func (cs *State) defaultDecideProposal(height int64, round int32) { var block *types.Block var blockParts *types.PartSet + var blockRows *types.RowSet // Decide on block if cs.ValidBlock != nil { // If there is valid block, choose that. - block, blockParts = cs.ValidBlock, cs.ValidBlockParts + block, blockParts, blockRows = cs.ValidBlock, cs.ValidBlockParts, cs.ValidBlockRows } else { // Create a new proposal block from state/txs from the mempool. - block, blockParts = cs.createProposalBlock() + block, blockParts, blockRows = cs.createProposalBlock() if block == nil { return } @@ -1121,9 +1126,9 @@ func (cs *State) defaultDecideProposal(height int64, round int32) { cs.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err) } - // cancel ctx for previous proposal block to ensure block putting/providing does not queues up - if cs.proposalCancel != nil { - // FIXME(ismail): below commented out cancel tries to prevent block putting + // cancel ctx for previous proposal block to ensure block providing does not queues up + if cs.provideCancel != nil { //nolint:staticcheck + // FIXME(ismail): below commented out cancel tries to prevent block providing // and providing no to queue up endlessly. // But in a real network proposers should have enough time in between. // And even if not, queuing up to a problematic extent will take a lot of time: @@ -1139,22 +1144,20 @@ func (cs *State) defaultDecideProposal(height int64, round int32) { // the provide timeout could still be larger than just the time between // two consecutive proposals. // - cs.proposalCancel() + cs.provideCancel() } - cs.proposalCtx, cs.proposalCancel = context.WithCancel(context.TODO()) - go func(ctx context.Context) { - cs.Logger.Info("Putting Block to IPFS", "height", block.Height) - err = ipld.PutBlock(ctx, cs.dag, block, cs.croute, cs.Logger) + cs.provideCtx, cs.provideCancel = context.WithCancel(context.TODO()) + go func(ctx context.Context, dah *ipld.DataAvailabilityHeader) { + err = ipld.ProvideData(ctx, dah, cs.croute, cs.Logger.With("height", block.Height)) if err != nil { if errors.Is(err, context.Canceled) { - cs.Logger.Error("Putting Block didn't finish in time and was terminated", "height", block.Height) + cs.Logger.Error("Providing Block didn't finish in time and was terminated", "height", block.Height) return } - cs.Logger.Error("Failed to put Block to IPFS", "err", err, "height", block.Height) + cs.Logger.Error("Failed to provide Block to DHT", "err", err, "height", block.Height) return } - cs.Logger.Info("Finished putting block to IPFS", "height", block.Height) - }(cs.proposalCtx) + }(cs.provideCtx, blockRows.DAHeader) } // Returns true if the proposal block is complete && @@ -1180,7 +1183,7 @@ func (cs *State) isProposalComplete() bool { // // NOTE: keep it side-effect free for clarity. // CONTRACT: cs.privValidator is not nil. -func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.PartSet) { +func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.PartSet, blockRows *types.RowSet) { if cs.privValidator == nil { panic("entered createProposalBlock with privValidator being nil") } @@ -1365,6 +1368,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil + cs.LockedBlockRows = nil if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { cs.Logger.Error("Error publishing event unlock", "err", err) } @@ -1413,6 +1417,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { if !cs.ProposalBlockParts.HasHeader(blockID.PartSetHeader) { cs.ProposalBlock = nil cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) + cs.ProposalBlockRows = nil } if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { cs.Logger.Error("Error publishing event unlock", "err", err) @@ -1487,6 +1492,7 @@ func (cs *State) enterCommit(height int64, commitRound int32) { logger.Info("Commit is for locked block. Set ProposalBlock=LockedBlock", "blockHash", blockID.Hash) cs.ProposalBlock = cs.LockedBlock cs.ProposalBlockParts = cs.LockedBlockParts + cs.ProposalBlockRows = cs.LockedBlockRows } // If we don't have the block being committed, set up to get it. @@ -1502,6 +1508,7 @@ func (cs *State) enterCommit(height int64, commitRound int32) { // Set up ProposalBlockParts and keep waiting. cs.ProposalBlock = nil cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) + cs.ProposalBlockRows = nil if err := cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()); err != nil { cs.Logger.Error("Error publishing valid block", "err", err) } @@ -1864,6 +1871,10 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.ID) (add } cs.ProposalBlock = block + cs.ProposalBlockRows, err = block.RowSet(context.TODO(), cs.dag) + if err != nil { + return false, err + } // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) if err := cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent()); err != nil { @@ -1880,6 +1891,7 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.ID) (add cs.ValidRound = cs.Round cs.ValidBlock = cs.ProposalBlock cs.ValidBlockParts = cs.ProposalBlockParts + cs.ValidBlockRows = cs.ProposalBlockRows } // TODO: In case there is +2/3 majority in Prevotes set for some // block and cs.ProposalBlock contains different block, either @@ -2046,6 +2058,7 @@ func (cs *State) addVote( cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil + cs.LockedBlockRows = nil if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { return added, err } @@ -2061,12 +2074,14 @@ func (cs *State) addVote( cs.ValidRound = vote.Round cs.ValidBlock = cs.ProposalBlock cs.ValidBlockParts = cs.ProposalBlockParts + cs.ValidBlockRows = cs.ProposalBlockRows } else { cs.Logger.Info( "Valid block we don't know about. Set ProposalBlock=nil", "proposal", cs.ProposalBlock.Hash(), "blockID", blockID.Hash) // We're getting the wrong block. cs.ProposalBlock = nil + cs.ProposalBlockRows = nil } if !cs.ProposalBlockParts.HasHeader(blockID.PartSetHeader) { cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) diff --git a/consensus/types/round_state.go b/consensus/types/round_state.go index 8c76019a44..b305565769 100644 --- a/consensus/types/round_state.go +++ b/consensus/types/round_state.go @@ -76,13 +76,16 @@ type RoundState struct { Proposal *types.Proposal `json:"proposal"` ProposalBlock *types.Block `json:"proposal_block"` ProposalBlockParts *types.PartSet `json:"proposal_block_parts"` + ProposalBlockRows *types.RowSet `json:"proposal_block_rows"` LockedRound int32 `json:"locked_round"` LockedBlock *types.Block `json:"locked_block"` LockedBlockParts *types.PartSet `json:"locked_block_parts"` + LockedBlockRows *types.RowSet `json:"locked_block_rows"` // Last known round with POL for non-nil valid block. - ValidRound int32 `json:"valid_round"` - ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above. + ValidRound int32 `json:"valid_round"` + ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above. + ValidBlockRows *types.RowSet `json:"valid_block_rows"` // Last known block parts of POL mentioned above. ValidBlockParts *types.PartSet `json:"valid_block_parts"` diff --git a/node/node.go b/node/node.go index 72b5d64aa4..c221f2f173 100644 --- a/node/node.go +++ b/node/node.go @@ -734,6 +734,7 @@ func NewNode(config *cfg.Config, mempool, evidencePool, sm.BlockExecutorWithMetrics(smMetrics), + sm.BlockExecutorWithDAG(ipfsNode.DAG), ) // Make BlockchainReactor. Don't start fast sync if we're doing a state sync first. diff --git a/state/execution.go b/state/execution.go index 2666b3f9c0..41ca3ab18c 100644 --- a/state/execution.go +++ b/state/execution.go @@ -6,6 +6,9 @@ import ( "fmt" "time" + format "github.com/ipfs/go-ipld-format" + mdutils "github.com/ipfs/go-merkledag/test" + abci "github.com/lazyledger/lazyledger-core/abci/types" cryptoenc "github.com/lazyledger/lazyledger-core/crypto/encoding" "github.com/lazyledger/lazyledger-core/libs/fail" @@ -24,6 +27,9 @@ import ( // BlockExecutor provides the context and accessories for properly executing a block. type BlockExecutor struct { + // DAG Store + adder format.NodeAdder + // save state, validators, consensus params, abci responses here store Store @@ -51,6 +57,12 @@ func BlockExecutorWithMetrics(metrics *Metrics) BlockExecutorOption { } } +func BlockExecutorWithDAG(adder format.NodeAdder) BlockExecutorOption { + return func(blockExec *BlockExecutor) { + blockExec.adder = adder + } +} + // NewBlockExecutor returns a new BlockExecutor with a NopEventBus. // Call SetEventBus to provide one. func NewBlockExecutor( @@ -74,7 +86,9 @@ func NewBlockExecutor( for _, option := range options { option(res) } - + if res.adder == nil { + res.adder = mdutils.Mock() + } return res } @@ -96,7 +110,7 @@ func (blockExec *BlockExecutor) CreateProposalBlock( height int64, state State, commit *types.Commit, proposerAddr []byte, -) (*types.Block, *types.PartSet) { +) (*types.Block, *types.PartSet, *types.RowSet) { maxBytes := state.ConsensusParams.Block.MaxBytes maxGas := state.ConsensusParams.Block.MaxGas @@ -155,7 +169,14 @@ func (blockExec *BlockExecutor) CreateProposalBlock( messages := types.MessagesFromProto(pbmessages) - return state.MakeBlock(height, processedTxs, evidence, nil, messages, commit, proposerAddr) + block := state.MakeBlock(height, processedTxs, evidence, nil, messages, commit, proposerAddr) + rows, err := block.RowSet(context.TODO(), blockExec.adder) + if err != nil { + blockExec.logger.Error("Can't make RowSet", "err", err) + return nil, nil, nil + } + + return block, block.MakePartSet(types.BlockPartSizeBytes), rows } // ValidateBlock validates the given block against the given state. diff --git a/state/state.go b/state/state.go index b3c4858b05..2c33977a2f 100644 --- a/state/state.go +++ b/state/state.go @@ -241,7 +241,7 @@ func (state State) MakeBlock( messages types.Messages, commit *types.Commit, proposerAddress []byte, -) (*types.Block, *types.PartSet) { +) *types.Block { // Build base block with block data. block := types.MakeBlock(height, txs, evidence, intermediateStateRoots, messages, commit) @@ -263,7 +263,7 @@ func (state State) MakeBlock( proposerAddress, ) - return block, block.MakePartSet(types.BlockPartSizeBytes) + return block } // MedianTime computes a median time for a given Commit (based on Timestamp field of votes messages) and the From 4fd6aa4afa53acaab851d07d0ee58b93f79b30c5 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Mon, 21 Jun 2021 14:53:25 +0300 Subject: [PATCH 09/11] fix: tests and build everywhere --- blockchain/msgs_test.go | 5 +++++ blockchain/v0/reactor_test.go | 8 ++++++-- consensus/byzantine_test.go | 4 ++-- consensus/common_test.go | 2 +- consensus/msgs_test.go | 7 ++++--- consensus/replay_test.go | 16 ++++++++++------ consensus/state_test.go | 4 ++-- evidence/pool_test.go | 9 +++++++-- light/client.go | 3 +-- light/provider/http/http.go | 3 ++- node/node_test.go | 6 +++--- privval/file_test.go | 3 ++- privval/msgs_test.go | 3 ++- privval/signer_client_test.go | 7 ++++--- rpc/core/types/responses.go | 3 ++- state/execution_test.go | 2 +- state/helpers_test.go | 13 ++++++++++--- state/validation_test.go | 12 +++++------- store/store_test.go | 8 ++++++-- tools/tm-signer-harness/internal/test_harness.go | 3 ++- 20 files changed, 77 insertions(+), 44 deletions(-) diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go index 3a608430e8..8c83d0ce01 100644 --- a/blockchain/msgs_test.go +++ b/blockchain/msgs_test.go @@ -1,11 +1,13 @@ package blockchain import ( + "context" "encoding/hex" "math" "testing" "github.com/gogo/protobuf/proto" + mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -83,6 +85,9 @@ func TestBlockchainMessageVectors(t *testing.T) { block := types.MakeBlock(int64(3), []types.Tx{types.Tx("Hello World")}, nil, nil, types.Messages{}, nil) block.Version.Block = 11 // overwrite updated protocol version + _, err := block.RowSet(context.TODO(), mdutils.Mock()) + require.NoError(t, err) + bpb, err := block.ToProto() require.NoError(t, err) diff --git a/blockchain/v0/reactor_test.go b/blockchain/v0/reactor_test.go index b87e2853fe..bc8c2313c0 100644 --- a/blockchain/v0/reactor_test.go +++ b/blockchain/v0/reactor_test.go @@ -302,9 +302,13 @@ func makeTxs(height int64) (txs []types.Tx) { } func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Block { - block, _ := state.MakeBlock(height, makeTxs(height), nil, + b := state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, lastCommit, state.Validators.GetProposer().Address) - return block + _, err := b.RowSet(context.TODO(), mdutils.Mock()) + if err != nil { + panic(err) + } + return b } type testApp struct { diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 821467b98c..1448e2a759 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -384,7 +384,7 @@ func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *St // Avoid sending on internalMsgQueue and running consensus state. // Create a new proposal block from state/txs from the mempool. - block1, blockParts1 := cs.createProposalBlock() + block1, blockParts1, _ := cs.createProposalBlock() polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash(), PartSetHeader: blockParts1.Header()} proposal1 := types.NewProposal(height, round, polRound, propBlockID, &block1.DataAvailabilityHeader) p1, err := proposal1.ToProto() @@ -399,7 +399,7 @@ func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *St deliverTxsRange(cs, 0, 1) // Create a new proposal block from state/txs from the mempool. - block2, blockParts2 := cs.createProposalBlock() + block2, blockParts2, _ := cs.createProposalBlock() polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash(), PartSetHeader: blockParts2.Header()} proposal2 := types.NewProposal(height, round, polRound, propBlockID, &block2.DataAvailabilityHeader) p2, err := proposal2.ToProto() diff --git a/consensus/common_test.go b/consensus/common_test.go index 2f41c2eb4e..589e2f26f8 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -196,7 +196,7 @@ func decideProposal( round int32, ) (proposal *types.Proposal, block *types.Block) { cs1.mtx.Lock() - block, blockParts := cs1.createProposalBlock() + block, blockParts, _ := cs1.createProposalBlock() validRound := cs1.ValidRound chainID := cs1.state.ChainID cs1.mtx.Unlock() diff --git a/consensus/msgs_test.go b/consensus/msgs_test.go index 1ca5c30b62..83a404b53e 100644 --- a/consensus/msgs_test.go +++ b/consensus/msgs_test.go @@ -15,6 +15,7 @@ import ( "github.com/lazyledger/lazyledger-core/libs/bits" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" "github.com/lazyledger/lazyledger-core/p2p" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmcons "github.com/lazyledger/lazyledger-core/proto/tendermint/consensus" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" "github.com/lazyledger/lazyledger-core/types" @@ -48,7 +49,7 @@ func TestMsgToProto(t *testing.T) { pbParts, err := parts.ToProto() require.NoError(t, err) - roots, err := types.NmtRootsFromBytes([][]byte{tmrand.Bytes(2*consts.NamespaceSize + tmhash.Size)}) + roots, err := ipld.NmtRootsFromBytes([][]byte{tmrand.Bytes(2*consts.NamespaceSize + tmhash.Size)}) require.NoError(t, err) proposal := types.Proposal{ Type: tmproto.ProposalType, @@ -58,7 +59,7 @@ func TestMsgToProto(t *testing.T) { BlockID: bi, Timestamp: time.Now(), Signature: tmrand.Bytes(20), - DAHeader: &types.DataAvailabilityHeader{ + DAHeader: &ipld.DataAvailabilityHeader{ RowsRoots: roots, ColumnRoots: roots, }, @@ -361,7 +362,7 @@ func TestConsMsgsVectors(t *testing.T) { BlockID: bi, Timestamp: date, Signature: []byte("add_more_exclamation"), - DAHeader: &types.DataAvailabilityHeader{}, + DAHeader: &ipld.DataAvailabilityHeader{}, } pbProposal, err := proposal.ToProto() require.NoError(t, err) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 8b367ae961..75ddc6cd66 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -368,7 +368,7 @@ func TestSimulateValidatorsChange(t *testing.T) { newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower) err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx1, nil, mempl.TxInfo{}) assert.Nil(t, err) - propBlock, _ := css[0].createProposalBlock() // changeProposer(t, cs1, vs2) + propBlock, _, _ := css[0].createProposalBlock() // changeProposer(t, cs1, vs2) propBlockParts := propBlock.MakePartSet(partSize) blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} @@ -399,7 +399,7 @@ func TestSimulateValidatorsChange(t *testing.T) { updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) err = assertMempool(css[0].txNotifier).CheckTx(updateValidatorTx1, nil, mempl.TxInfo{}) assert.Nil(t, err) - propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) + propBlock, _, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) propBlockParts = propBlock.MakePartSet(partSize) blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} @@ -437,7 +437,7 @@ func TestSimulateValidatorsChange(t *testing.T) { newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower) err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx3, nil, mempl.TxInfo{}) assert.Nil(t, err) - propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) + propBlock, _, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) propBlockParts = propBlock.MakePartSet(partSize) blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} newVss := make([]*validatorStub, nVals+1) @@ -513,7 +513,7 @@ func TestSimulateValidatorsChange(t *testing.T) { removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0) err = assertMempool(css[0].txNotifier).CheckTx(removeValidatorTx3, nil, mempl.TxInfo{}) assert.Nil(t, err) - propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) + propBlock, _, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) propBlockParts = propBlock.MakePartSet(partSize) blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} newVss = make([]*validatorStub, nVals+3) @@ -1001,8 +1001,7 @@ func makeBlock(state sm.State, lastBlock *types.Block, lastBlockMeta *types.Bloc lastCommit = types.NewCommit(vote.Height, vote.Round, lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()}) } - - return state.MakeBlock( + block := state.MakeBlock( height, []types.Tx{}, nil, @@ -1011,6 +1010,11 @@ func makeBlock(state sm.State, lastBlock *types.Block, lastBlockMeta *types.Bloc lastCommit, state.Validators.GetProposer().Address, ) + _, err := block.RowSet(context.TODO(), mdutils.Mock()) + if err != nil { + panic(err) + } + return block, block.MakePartSet(types.BlockPartSizeBytes) } type badApp struct { diff --git a/consensus/state_test.go b/consensus/state_test.go index e713917dd5..cd47dc1fc9 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -194,7 +194,7 @@ func TestStateBadProposal(t *testing.T) { proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) voteCh := subscribe(cs1.eventBus, types.EventQueryVote) - propBlock, _ := cs1.createProposalBlock() // changeProposer(t, cs1, vs2) + propBlock, _, _ := cs1.createProposalBlock() // changeProposer(t, cs1, vs2) // make the second validator the proposer by incrementing round round++ @@ -255,7 +255,7 @@ func TestStateOversizedBlock(t *testing.T) { timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) voteCh := subscribe(cs1.eventBus, types.EventQueryVote) - propBlock, _ := cs1.createProposalBlock() + propBlock, _, _ := cs1.createProposalBlock() propBlock.Data.Txs = []types.Tx{tmrand.Bytes(2001)} propBlock.Header.DataHash = propBlock.DataAvailabilityHeader.Hash() diff --git a/evidence/pool_test.go b/evidence/pool_test.go index 8fb837afde..d4e26f850e 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -175,6 +175,8 @@ func TestEvidencePoolUpdate(t *testing.T) { val, evidenceChainID) lastCommit := makeCommit(height, val.PrivKey.PubKey().Address()) block := types.MakeBlock(height+1, []types.Tx{}, []types.Evidence{ev}, nil, types.Messages{}, lastCommit) + _, err = block.RowSet(context.TODO(), mdutils.Mock()) + require.NoError(t, err) // update state (partially) state.LastBlockHeight = height + 1 state.LastBlockTime = defaultEvidenceTime.Add(22 * time.Minute) @@ -400,13 +402,16 @@ func initializeBlockStore(db dbm.DB, state sm.State, valAddr []byte) *store.Bloc for i := int64(1); i <= state.LastBlockHeight; i++ { lastCommit := makeCommit(i-1, valAddr) - block, _ := state.MakeBlock(i, []types.Tx{}, nil, nil, + block := state.MakeBlock(i, []types.Tx{}, nil, nil, types.Messages{}, lastCommit, state.Validators.GetProposer().Address) block.Header.Time = defaultEvidenceTime.Add(time.Duration(i) * time.Minute) block.Header.Version = tmversion.Consensus{Block: version.BlockProtocol, App: 1} const parts = 1 partSet := block.MakePartSet(parts) - + _, err := block.RowSet(context.TODO(), mdutils.Mock()) + if err != nil { + panic(err) + } seenCommit := makeCommit(i, valAddr) err := blockStore.SaveBlock(context.TODO(), block, partSet, seenCommit) if err != nil { diff --git a/light/client.go b/light/client.go index fca018058a..cecfc429fa 100644 --- a/light/client.go +++ b/light/client.go @@ -9,7 +9,6 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" - "github.com/lazyledger/nmt/namespace" "github.com/lazyledger/lazyledger-core/libs/log" tmmath "github.com/lazyledger/lazyledger-core/libs/math" @@ -700,7 +699,7 @@ func (c *Client) verifySequential( c.dag, interimBlock.DataAvailabilityHeader, numSamples, - func(data namespace.PrefixedData8) {}, // noop + func(ipld.NamespacedShare) {}, // noop ) if err != nil { return fmt.Errorf("data availability sampling failed; ipld.ValidateAvailability: %w", err) diff --git a/light/provider/http/http.go b/light/provider/http/http.go index 566f4f85a6..c52e0db37c 100644 --- a/light/provider/http/http.go +++ b/light/provider/http/http.go @@ -10,6 +10,7 @@ import ( "time" "github.com/lazyledger/lazyledger-core/light/provider" + "github.com/lazyledger/lazyledger-core/p2p/ipld" rpcclient "github.com/lazyledger/lazyledger-core/rpc/client" rpchttp "github.com/lazyledger/lazyledger-core/rpc/client/http" "github.com/lazyledger/lazyledger-core/types" @@ -181,7 +182,7 @@ func (p *http) signedHeader(ctx context.Context, height *int64) (*types.SignedHe return nil, provider.ErrNoResponse } -func (p *http) daHeader(ctx context.Context, height *int64) (*types.DataAvailabilityHeader, error) { +func (p *http) daHeader(ctx context.Context, height *int64) (*ipld.DataAvailabilityHeader, error) { for attempt := 1; attempt <= maxRetryAttempts; attempt++ { daHeaderRes, err := p.client.DataAvailabilityHeader(ctx, height) if err != nil { diff --git a/node/node_test.go b/node/node_test.go index 92b00fd3f3..4ef0fdf1a7 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -321,7 +321,7 @@ func TestCreateProposalBlock(t *testing.T) { ) commit := types.NewCommit(height-1, 0, types.BlockID{}, nil) - block, _ := blockExec.CreateProposalBlock( + block, _, _ := blockExec.CreateProposalBlock( height, state, commit, proposerAddr, @@ -390,7 +390,7 @@ func TestMaxTxsProposalBlockSize(t *testing.T) { ) commit := types.NewCommit(height-1, 0, types.BlockID{}, nil) - block, _ := blockExec.CreateProposalBlock( + block, _, _ := blockExec.CreateProposalBlock( height, state, commit, proposerAddr, @@ -497,7 +497,7 @@ func TestMaxProposalBlockSize(t *testing.T) { commit.Signatures = append(commit.Signatures, cs) } - block, partSet := blockExec.CreateProposalBlock( + block, partSet, _ := blockExec.CreateProposalBlock( math.MaxInt64, state, commit, proposerAddr, diff --git a/privval/file_test.go b/privval/file_test.go index 9ddde34c84..ebaa56bfb9 100644 --- a/privval/file_test.go +++ b/privval/file_test.go @@ -15,6 +15,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto/tmhash" tmjson "github.com/lazyledger/lazyledger-core/libs/json" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" "github.com/lazyledger/lazyledger-core/types" tmtime "github.com/lazyledger/lazyledger-core/types/time" @@ -350,6 +351,6 @@ func newProposal(height int64, round int32, blockID types.BlockID) *types.Propos Round: round, BlockID: blockID, Timestamp: tmtime.Now(), - DAHeader: &types.DataAvailabilityHeader{}, + DAHeader: &ipld.DataAvailabilityHeader{}, } } diff --git a/privval/msgs_test.go b/privval/msgs_test.go index a11e4221df..35eca0d725 100644 --- a/privval/msgs_test.go +++ b/privval/msgs_test.go @@ -12,6 +12,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto/ed25519" cryptoenc "github.com/lazyledger/lazyledger-core/crypto/encoding" "github.com/lazyledger/lazyledger-core/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/p2p/ipld" cryptoproto "github.com/lazyledger/lazyledger-core/proto/tendermint/crypto" privproto "github.com/lazyledger/lazyledger-core/proto/tendermint/privval" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" @@ -54,7 +55,7 @@ func exampleProposal() *types.Proposal { Hash: tmhash.Sum([]byte("blockID_part_set_header_hash")), }, }, - DAHeader: &types.DataAvailabilityHeader{}, + DAHeader: &ipld.DataAvailabilityHeader{}, } } diff --git a/privval/signer_client_test.go b/privval/signer_client_test.go index 645e7ecb97..c51b483526 100644 --- a/privval/signer_client_test.go +++ b/privval/signer_client_test.go @@ -11,6 +11,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/tmhash" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" + "github.com/lazyledger/lazyledger-core/p2p/ipld" cryptoproto "github.com/lazyledger/lazyledger-core/proto/tendermint/crypto" privvalproto "github.com/lazyledger/lazyledger-core/proto/tendermint/privval" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" @@ -125,7 +126,7 @@ func TestSignerProposal(t *testing.T) { POLRound: 2, BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, Timestamp: ts, - DAHeader: &types.DataAvailabilityHeader{}, + DAHeader: &ipld.DataAvailabilityHeader{}, } want := &types.Proposal{ Type: tmproto.ProposalType, @@ -134,7 +135,7 @@ func TestSignerProposal(t *testing.T) { POLRound: 2, BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, Timestamp: ts, - DAHeader: &types.DataAvailabilityHeader{}, + DAHeader: &ipld.DataAvailabilityHeader{}, } tc := tc @@ -342,7 +343,7 @@ func TestSignerSignProposalErrors(t *testing.T) { BlockID: types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}}, Timestamp: ts, Signature: []byte("signature"), - DAHeader: &types.DataAvailabilityHeader{}, + DAHeader: &ipld.DataAvailabilityHeader{}, } p, err := proposal.ToProto() diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index bb206333f2..a191079ae2 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -8,6 +8,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/libs/bytes" "github.com/lazyledger/lazyledger-core/p2p" + "github.com/lazyledger/lazyledger-core/p2p/ipld" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" "github.com/lazyledger/lazyledger-core/types" ) @@ -36,7 +37,7 @@ type ResultCommit struct { } type ResultDataAvailabilityHeader struct { - types.DataAvailabilityHeader `json:"data_availability_header"` + ipld.DataAvailabilityHeader `json:"data_availability_header"` } // ABCI results from a block diff --git a/state/execution_test.go b/state/execution_test.go index b6ea333d14..5c393746fd 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -100,7 +100,7 @@ func TestBeginBlockValidators(t *testing.T) { lastCommit := types.NewCommit(1, 0, prevBlockID, tc.lastCommitSigs) // block for height 2 - block, _ := state.MakeBlock(2, makeTxs(2), nil, nil, + block := state.MakeBlock(2, makeTxs(2), nil, nil, types.Messages{}, lastCommit, state.Validators.GetProposer().Address) _, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, log.TestingLogger(), stateStore, 1) diff --git a/state/helpers_test.go b/state/helpers_test.go index 32a3c78cf8..5a713244ea 100644 --- a/state/helpers_test.go +++ b/state/helpers_test.go @@ -2,9 +2,12 @@ package state_test import ( "bytes" + "context" "fmt" "time" + mdutils "github.com/ipfs/go-merkledag/test" + abci "github.com/lazyledger/lazyledger-core/abci/types" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/ed25519" @@ -54,7 +57,7 @@ func makeAndCommitGoodBlock( func makeAndApplyGoodBlock(state sm.State, height int64, lastCommit *types.Commit, proposerAddr []byte, blockExec *sm.BlockExecutor, evidence []types.Evidence) (sm.State, types.BlockID, error) { - block, _ := state.MakeBlock( + block := state.MakeBlock( height, makeTxs(height), evidence, @@ -63,12 +66,16 @@ func makeAndApplyGoodBlock(state sm.State, height int64, lastCommit *types.Commi lastCommit, proposerAddr, ) + _, err := block.RowSet(context.TODO(), mdutils.Mock()) + if err != nil { + return sm.State{}, types.BlockID{}, err + } if err := blockExec.ValidateBlock(state, block); err != nil { return state, types.BlockID{}, err } blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: types.PartSetHeader{Total: 3, Hash: tmrand.Bytes(32)}} - state, _, err := blockExec.ApplyBlock(state, blockID, block) + state, _, err = blockExec.ApplyBlock(state, blockID, block) if err != nil { return state, types.BlockID{}, err } @@ -140,7 +147,7 @@ func makeState(nVals, height int) (sm.State, dbm.DB, map[string]types.PrivValida } func makeBlock(state sm.State, height int64) *types.Block { - block, _ := state.MakeBlock( + block := state.MakeBlock( height, makeTxs(state.LastBlockHeight), nil, diff --git a/state/validation_test.go b/state/validation_test.go index 1aa1853d60..cd2e9f5a04 100644 --- a/state/validation_test.go +++ b/state/validation_test.go @@ -59,8 +59,6 @@ func TestValidateBlockHeader(t *testing.T) { {"LastBlockID wrong", func(block *types.Block) { block.LastBlockID.PartSetHeader.Total += 10 }}, {"LastCommitHash wrong", func(block *types.Block) { block.LastCommitHash = wrongHash }}, - {"DataHash wrong", func(block *types.Block) { block.DataHash = wrongHash }}, - {"ValidatorsHash wrong", func(block *types.Block) { block.ValidatorsHash = wrongHash }}, {"NextValidatorsHash wrong", func(block *types.Block) { block.NextValidatorsHash = wrongHash }}, {"ConsensusHash wrong", func(block *types.Block) { block.ConsensusHash = wrongHash }}, @@ -84,7 +82,7 @@ func TestValidateBlockHeader(t *testing.T) { Invalid blocks don't pass */ for _, tc := range testCases { - block, _ := state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, lastCommit, proposerAddr) + block := state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, lastCommit, proposerAddr) tc.malleateBlock(block) err := blockExec.ValidateBlock(state, block) t.Logf("%s: %v", tc.name, err) @@ -101,7 +99,7 @@ func TestValidateBlockHeader(t *testing.T) { } nextHeight := validationTestsStopHeight - block, _ := state.MakeBlock( + block := state.MakeBlock( nextHeight, makeTxs(nextHeight), nil, nil, types.Messages{}, lastCommit, @@ -153,7 +151,7 @@ func TestValidateBlockCommit(t *testing.T) { state.LastBlockID, []types.CommitSig{wrongHeightVote.CommitSig()}, ) - block, _ := state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, wrongHeightCommit, proposerAddr) + block := state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, wrongHeightCommit, proposerAddr) err = blockExec.ValidateBlock(state, block) _, isErrInvalidCommitHeight := err.(types.ErrInvalidCommitHeight) require.True(t, isErrInvalidCommitHeight, "expected ErrInvalidCommitHeight at height %d but got: %v", height, err) @@ -161,7 +159,7 @@ func TestValidateBlockCommit(t *testing.T) { /* #2589: test len(block.LastCommit.Signatures) == state.LastValidators.Size() */ - block, _ = state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, wrongSigsCommit, proposerAddr) + block = state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, wrongSigsCommit, proposerAddr) err = blockExec.ValidateBlock(state, block) _, isErrInvalidCommitSignatures := err.(types.ErrInvalidCommitSignatures) require.True(t, isErrInvalidCommitSignatures, @@ -270,7 +268,7 @@ func TestValidateBlockEvidence(t *testing.T) { evidence = append(evidence, newEv) currentBytes += int64(len(newEv.Bytes())) } - block, _ := state.MakeBlock(height, makeTxs(height), evidence, nil, types.Messages{}, lastCommit, proposerAddr) + block := state.MakeBlock(height, makeTxs(height), evidence, nil, types.Messages{}, lastCommit, proposerAddr) err := blockExec.ValidateBlock(state, block) if assert.Error(t, err) { _, ok := err.(*types.ErrEvidenceOverflow) diff --git a/store/store_test.go b/store/store_test.go index c54b78937b..518256bf93 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -58,9 +58,13 @@ func makeTxs(height int64) (txs []types.Tx) { } func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Block { - block, _ := state.MakeBlock(height, makeTxs(height), nil, + b := state.MakeBlock(height, makeTxs(height), nil, nil, types.Messages{}, lastCommit, state.Validators.GetProposer().Address) - return block + _, err := b.RowSet(context.TODO(), mdutils.Mock()) + if err != nil { + panic(err) + } + return b } func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore, cleanupFunc) { diff --git a/tools/tm-signer-harness/internal/test_harness.go b/tools/tm-signer-harness/internal/test_harness.go index e2a4d01532..b7656d38ab 100644 --- a/tools/tm-signer-harness/internal/test_harness.go +++ b/tools/tm-signer-harness/internal/test_harness.go @@ -9,6 +9,7 @@ import ( "time" "github.com/lazyledger/lazyledger-core/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/p2p/ipld" "github.com/lazyledger/lazyledger-core/crypto/ed25519" "github.com/lazyledger/lazyledger-core/privval" @@ -228,7 +229,7 @@ func (th *TestHarness) TestSignProposal() error { }, }, Timestamp: time.Now(), - DAHeader: &types.DataAvailabilityHeader{}, + DAHeader: &ipld.DataAvailabilityHeader{}, } p, err := prop.ToProto() if err != nil { From c9d932ad325a685ad6e108e61173fada72e69ada Mon Sep 17 00:00:00 2001 From: Wondertan Date: Sun, 27 Jun 2021 22:47:21 +0300 Subject: [PATCH 10/11] fix(state): update to new p2p/ipld API --- blockchain/v0/reactor_test.go | 1 + consensus/state.go | 2 +- evidence/pool_test.go | 3 ++- rpc/core/blocks.go | 3 ++- store/store.go | 20 +++++++++++++------- store/store_test.go | 1 + 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/blockchain/v0/reactor_test.go b/blockchain/v0/reactor_test.go index bc8c2313c0..c2ff57aaf9 100644 --- a/blockchain/v0/reactor_test.go +++ b/blockchain/v0/reactor_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/consensus/state.go b/consensus/state.go index c7c443f0fe..02a7e9fb7e 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1127,7 +1127,7 @@ func (cs *State) defaultDecideProposal(height int64, round int32) { } // cancel ctx for previous proposal block to ensure block providing does not queues up - if cs.provideCancel != nil { //nolint:staticcheck + if cs.provideCancel != nil { // FIXME(ismail): below commented out cancel tries to prevent block providing // and providing no to queue up endlessly. // But in a real network proposers should have enough time in between. diff --git a/evidence/pool_test.go b/evidence/pool_test.go index d4e26f850e..7c3b161e1a 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -413,7 +414,7 @@ func initializeBlockStore(db dbm.DB, state sm.State, valAddr []byte) *store.Bloc panic(err) } seenCommit := makeCommit(i, valAddr) - err := blockStore.SaveBlock(context.TODO(), block, partSet, seenCommit) + err = blockStore.SaveBlock(context.TODO(), block, partSet, seenCommit) if err != nil { panic(err) } diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go index 06d411a6a3..a98f8adb15 100644 --- a/rpc/core/blocks.go +++ b/rpc/core/blocks.go @@ -4,6 +4,7 @@ import ( "fmt" tmmath "github.com/lazyledger/lazyledger-core/libs/math" + "github.com/lazyledger/lazyledger-core/p2p/ipld" ctypes "github.com/lazyledger/lazyledger-core/rpc/core/types" rpctypes "github.com/lazyledger/lazyledger-core/rpc/jsonrpc/types" "github.com/lazyledger/lazyledger-core/types" @@ -156,7 +157,7 @@ func DataAvailabilityHeader(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.Re block, err := env.BlockStore.LoadBlock(ctx.Context(), height) if err != nil { return &ctypes.ResultDataAvailabilityHeader{ - DataAvailabilityHeader: types.DataAvailabilityHeader{}, + DataAvailabilityHeader: ipld.DataAvailabilityHeader{}, }, err } _ = block.Hash() diff --git a/store/store.go b/store/store.go index 2e6c684695..109a64b196 100644 --- a/store/store.go +++ b/store/store.go @@ -3,9 +3,8 @@ package store import ( "context" "fmt" - "strings" - "strconv" + "strings" "github.com/gogo/protobuf/proto" "github.com/ipfs/go-blockservice" @@ -14,7 +13,8 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" - "github.com/lazyledger/lazyledger-core/ipfs" + "github.com/lazyledger/rsmt2d" + dbm "github.com/lazyledger/lazyledger-core/libs/db" "github.com/lazyledger/lazyledger-core/libs/log" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" @@ -22,7 +22,6 @@ import ( tmstore "github.com/lazyledger/lazyledger-core/proto/tendermint/store" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" "github.com/lazyledger/lazyledger-core/types" - "github.com/lazyledger/rsmt2d" ) /* @@ -115,7 +114,7 @@ func (bs *BlockStore) LoadBlock(ctx context.Context, height int64) (*types.Block lastCommit := bs.LoadBlockCommit(height - 1) - data, err := ipld.RetrieveBlockData(ctx, &blockMeta.DAHeader, bs.dag, rsmt2d.NewRSGF8Codec()) + data, err := ipld.RetrieveData(ctx, &blockMeta.DAHeader, bs.dag, rsmt2d.NewRSGF8Codec()) if err != nil { if strings.Contains(err.Error(), format.ErrNotFound.Error()) { return nil, fmt.Errorf("failure to retrieve block data from local ipfs store: %w", err) @@ -124,9 +123,14 @@ func (bs *BlockStore) LoadBlock(ctx context.Context, height int64) (*types.Block return nil, err } + bdata, err := types.DataFromSquare(data) + if err != nil { + return nil, err + } + block := types.Block{ Header: blockMeta.Header, - Data: data, + Data: bdata, DataAvailabilityHeader: blockMeta.DAHeader, LastCommit: lastCommit, } @@ -369,13 +373,15 @@ func (bs *BlockStore) SaveBlock( bs.saveBlockPart(height, i, part) } - err := ipld.PutBlock(ctx, bs.dag, block, ipfs.MockRouting(), bs.logger) + shares, _ := block.ComputeShares() + eds, err := ipld.PutData(ctx, shares, bs.dag) if err != nil { return err } // Save block meta blockMeta := types.NewBlockMeta(block, blockParts) + blockMeta.DAHeader = *ipld.MakeDataHeader(eds) pbm, err := blockMeta.ToProto() if err != nil { panic(fmt.Errorf("failed to marshal block meta while saving: %w", err)) diff --git a/store/store_test.go b/store/store_test.go index 518256bf93..185ec46c47 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gogo/protobuf/proto" + mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" From a74bc0155f76d6ec6905f5a8934c6b047ebaf0bb Mon Sep 17 00:00:00 2001 From: Wondertan Date: Fri, 9 Jul 2021 11:10:35 +0300 Subject: [PATCH 11/11] fix(types): part set back to 5 parts --- types/block_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/types/block_test.go b/types/block_test.go index 8238472aac..1fe4d1138a 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -142,9 +142,12 @@ func TestBlockMakePartSetWithEvidence(t *testing.T) { ev := NewMockDuplicateVoteEvidenceWithValidator(h, time.Now(), vals[0], "block-test-chain") evList := []Evidence{ev} - partSet := MakeBlock(h, []Tx{Tx("Hello World")}, evList, nil, Messages{}, commit).MakePartSet(512) + block := MakeBlock(h, []Tx{Tx("Hello World")}, evList, nil, Messages{}, commit) + _, err = block.RowSet(context.TODO(), mdutils.Mock()) + require.NoError(t, err) + partSet := block.MakePartSet(512) assert.NotNil(t, partSet) - assert.EqualValues(t, 4, partSet.Total()) + assert.EqualValues(t, 5, partSet.Total()) } func TestBlockHashesTo(t *testing.T) {