diff --git a/Makefile b/Makefile index 465e56ce8..12338ed37 100644 --- a/Makefile +++ b/Makefile @@ -306,6 +306,11 @@ test_sortition: test_persistence: go test ${VERBOSE_TEST} -p=1 -count=1 ./persistence/... +.PHONY: test_persistence_state_hash +## Run all go unit tests in the Persistence module +test_persistence_state_hash: + go test -run StateHash ${VERBOSE_TEST} -p=1 ./persistence/... + .PHONY: test_p2p_types ## Run p2p subcomponents' tests test_p2p_types: diff --git a/go.mod b/go.mod index 54191cb69..f1507cdd9 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( ) require ( + github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 github.com/dgraph-io/badger/v3 v3.2103.2 github.com/jackc/pgconn v1.11.0 github.com/jordanorelli/lexnum v0.0.0-20141216151731-460eeb125754 @@ -40,7 +41,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.3 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index d064b7f46..8c2cca931 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 h1:iRNKw2WmAbVgGMNYzDH5Y2yY3+jyxwEK9Hc5pwIjZAE= +github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884/go.mod h1:/sdYDakowo/XaxS2Fl7CBqtuf/O2uTqF2zmAUFAtAiw= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/persistence/application.go b/persistence/application.go index 0f2b88512..cc5a8ed72 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -4,7 +4,9 @@ import ( "encoding/hex" "log" + "github.com/golang/protobuf/proto" "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/shared/modules" ) @@ -12,8 +14,57 @@ func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool return p.GetExists(types.ApplicationActor, address, height) } +func (p PostgresContext) UpdateAppTree(apps [][]byte) error { + for _, app := range apps { + appProto := types.Actor{} + if err := proto.Unmarshal(app, &appProto); err != nil { + return err + } + bzAddr, _ := hex.DecodeString(appProto.Address) + if _, err := p.MerkleTrees[AppMerkleTree].Update(bzAddr, app); err != nil { + return err + } + } + return nil +} + +func (p PostgresContext) getAppsUpdated(height int64) (apps []*types.Actor, err error) { + actors, err := p.GetActorsUpdated(types.ApplicationActor, height) + if err != nil { + return nil, err + } + + for _, actor := range actors { + // DISCUSS_IN_THIS_COMMIT: This breaks the pattern of protos in persistence. + // - Is it okay? + // - Do we embed this logic in `UpdateAppTree` + app := &types.Actor{ + Address: actor.Address, + PublicKey: actor.PublicKey, + // Paused: actor.Paused, // DISCUSS_IN_THIS_COMMIT: Is this just a check for pause height = -1? + // Status: actor.Status, // TODO_IN_THIS_COMMIT: Use logic from `GetActorStatus` without an extra query + Chains: actor.Chains, + // MaxRelays: actor.ActorSpecificParam, + // StakedTokens: actor.StakedTokens, + PausedHeight: actor.PausedHeight, + UnstakingHeight: actor.UnstakingHeight, + Output: actor.OutputAddress, + } + // appBytes, err := proto.Marshal(&app) + // if err != nil { + // return nil, err + // } + // apps = append(apps, appBytes) + apps = append(apps, app) + } + return +} + func (p PostgresContext) GetApp(address []byte, height int64) (operator, publicKey, stakedTokens, maxRelays, outputAddress string, pauseHeight, unstakingHeight int64, chains []string, err error) { actor, err := p.GetActor(types.ApplicationActor, address, height) + if err != nil { + return + } operator = actor.Address publicKey = actor.PublicKey stakedTokens = actor.StakedTokens diff --git a/persistence/block.go b/persistence/block.go index 170f81984..a6ba799a8 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -3,8 +3,9 @@ package persistence import ( "encoding/binary" "encoding/hex" - "github.com/pokt-network/pocket/persistence/types" "log" + + "github.com/pokt-network/pocket/persistence/types" ) // OPTIMIZE(team): get from blockstore or keep in memory @@ -52,7 +53,7 @@ func (p PostgresContext) StoreBlock(blockProtoBytes []byte) error { // INVESTIGATE: Note that we are writing this directly to the blockStore. Depending on how // the use of the PostgresContext evolves, we may need to write this to `ContextStore` and copy // over to `BlockStore` when the block is committed. - return p.DB.Blockstore.Put(heightToBytes(p.Height), blockProtoBytes) + return p.DB.Blockstore.Set(heightToBytes(p.Height), blockProtoBytes) } func (p PostgresContext) InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error { diff --git a/persistence/context.go b/persistence/context.go index fc2d7f892..da6ae9117 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -15,9 +15,17 @@ func (p PostgresContext) RollbackToSavePoint(bytes []byte) error { return p.DB.Tx.Rollback(context.TODO()) } +func (p PostgresContext) UpdateAppHash() ([]byte, error) { + if _, err := p.updateStateHash(); err != nil { + return nil, err + } + return p.StateHash, nil +} + func (p PostgresContext) AppHash() ([]byte, error) { - log.Println("TODO: AppHash not implemented") - return []byte("A real app hash, I am not"), nil + // log.Println("TODO: AppHash not implemented") + // return []byte("A real app hash, I am not"), n + return p.StateHash, nil } func (p PostgresContext) Reset() error { @@ -27,6 +35,11 @@ func (p PostgresContext) Reset() error { func (p PostgresContext) Commit() error { log.Printf("About to commit context at height %d.\n", p.Height) + // HACK: The data has already been written to the postgres DB, so what should we do here? The idea I have is: + // if _, err := p.updateStateHash(); err != nil { + // return err + // } + ctx := context.TODO() if err := p.DB.Tx.Commit(context.TODO()); err != nil { return err diff --git a/persistence/db.go b/persistence/db.go index 519e17670..60526a3ed 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/pokt-network/pocket/persistence/types" + "github.com/celestiaorg/smt" "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" "github.com/pokt-network/pocket/persistence/kvstore" @@ -42,6 +43,15 @@ var _ modules.PersistenceRWContext = &PostgresContext{} type PostgresContext struct { Height int64 DB PostgresDB + + ContextStore kvstore.KVStore + StateHash []byte + + // IMPROVE: Depending on how the use of `PostgresContext` evolves, we may be able to get + // access to these directly via the postgres module. + PostgresDB *pgx.Conn + BlockStore kvstore.KVStore // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the persistence module (i.e. not context) + MerkleTrees map[MerkleTree]*smt.SparseMerkleTree // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the persistence module (i.e. not context) } type PostgresDB struct { conn *pgx.Conn diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index 1b5fe2495..b2b11a5bd 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -1,33 +1,48 @@ package kvstore import ( + "errors" "log" + "github.com/celestiaorg/smt" badger "github.com/dgraph-io/badger/v3" ) -// CLEANUP: move this structure to a shared module +// TODO_IN_THIS_COMMIT: We might be able to remove the `KVStore` interface altogether if we end up using smt.MapStore type KVStore interface { // Lifecycle methods Stop() error // Accessors // TODO: Add a proper iterator interface - Put(key []byte, value []byte) error - Get(key []byte) ([]byte, error) // TODO: Add pagination for `GetAll` GetAll(prefixKey []byte, descending bool) ([][]byte, error) + Exists(key []byte) (bool, error) ClearAll() error + + // Same interface as in `smt.MapStore`` + Set(key []byte, value []byte) error + Get(key []byte) ([]byte, error) + Delete(key []byte) error } var _ KVStore = &badgerKVStore{} +var _ smt.MapStore = &badgerKVStore{} + +var ( + ErrKVStoreExists = errors.New("kvstore already exists") + ErrKVStoreNotExists = errors.New("kvstore does not exist") +) type badgerKVStore struct { db *badger.DB } -func NewKVStore(path string) (KVStore, error) { +// REFACTOR: Loads or creates a badgerDb at `path`. This may potentially need to be refactored +// into `NewKVStore` and `LoadKVStore` depending on how state sync evolves by leveraging `os.Stat` +// on the file path. +func OpenKVStore(path string) (KVStore, error) { db, err := badger.Open(badger.DefaultOptions(path)) if err != nil { return nil, err @@ -43,7 +58,7 @@ func NewMemKVStore() KVStore { return badgerKVStore{db: db} } -func (store badgerKVStore) Put(key []byte, value []byte) error { +func (store badgerKVStore) Set(key []byte, value []byte) error { txn := store.db.NewTransaction(true) defer txn.Discard() @@ -76,6 +91,11 @@ func (store badgerKVStore) Get(key []byte) ([]byte, error) { return value, nil } +func (store badgerKVStore) Delete(key []byte) error { + log.Fatalf("badgerKVStore.Delete not implemented yet") + return nil +} + func (store badgerKVStore) GetAll(prefix []byte, descending bool) (values [][]byte, err error) { // INVESTIGATE: research `badger.views` for further improvements and optimizations txn := store.db.NewTransaction(false) diff --git a/persistence/module.go b/persistence/module.go index a7e434735..a8ece5bae 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -7,10 +7,10 @@ import ( "io/ioutil" "log" - "github.com/pokt-network/pocket/persistence/types" - + "github.com/celestiaorg/smt" "github.com/jackc/pgx/v4" "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/test_artifacts" ) @@ -26,10 +26,19 @@ type PersistenceModule struct { postgresURL string nodeSchema string genesisPath string - blockStore kvstore.KVStore // INVESTIGATE: We may need to create a custom `BlockStore` package in the future // TECHDEBT: Need to implement context pooling (for writes), timeouts (for read & writes), etc... writeContext *PostgresContext // only one write context is allowed at a time + + // The connection to the PostgreSQL database + postgresConn *pgx.Conn + // A reference to the block key-value store + // INVESTIGATE: We may need to create a custom `BlockStore` package in the future. + blockStore kvstore.KVStore + // A mapping of context IDs to persistence contexts + // contexts map[contextId]modules.PersistenceRWContext + // Merkle trees + trees map[MerkleTree]*smt.SparseMerkleTree } const ( @@ -42,8 +51,10 @@ func Create(configPath, genesisPath string) (modules.PersistenceModule, error) { if err != nil { return nil, err } + cfg := c.(*types.PersistenceConfig) g, err := m.InitGenesis(genesisPath) + if err != nil { return nil, err } @@ -69,8 +80,20 @@ func Create(configPath, genesisPath string) (modules.PersistenceModule, error) { genesisPath: genesisPath, blockStore: blockStore, writeContext: nil, + // contexts: make(map[contextId]modules.PersistenceContext), + trees: make(map[MerkleTree]*smt.SparseMerkleTree), } + // DISCUSS_IN_THIS_COMMIT: We've been using the module function pattern, but this making `initializeTrees` + // be able to create and/or load trees outside the scope of the persistence module makes it easier to test. + trees, err := newMerkleTrees() + if err != nil { + return nil, err + } + + // TODO_IN_THIS_COMMIT: load trees from state + persistenceMod.trees = trees + // Determine if we should hydrate the genesis db or use the current state of the DB attached if shouldHydrateGenesis, err := persistenceMod.shouldHydrateGenesisDb(); err != nil { return nil, err @@ -218,7 +241,7 @@ func initializeBlockStore(blockStorePath string) (kvstore.KVStore, error) { if blockStorePath == "" { return kvstore.NewMemKVStore(), nil } - return kvstore.NewKVStore(blockStorePath) + return kvstore.OpenKVStore(blockStorePath) } // TODO(drewsky): Simplify and externalize the logic for whether genesis should be populated and diff --git a/persistence/shared_sql.go b/persistence/shared_sql.go index 21c35ce2a..1721ce884 100644 --- a/persistence/shared_sql.go +++ b/persistence/shared_sql.go @@ -44,6 +44,65 @@ func (p *PostgresContext) GetExists(actorSchema types.ProtocolActorSchema, addre return } +// TODO_IN_THIS_COMMIT: Consolidate logic with `GetActor` to reduce code footprint +func (p *PostgresContext) GetActorsUpdated(actorSchema types.ProtocolActorSchema, height int64) (actors []types.BaseActor, err error) { + ctx, tx, err := p.DB.GetCtxAndTxn() + if err != nil { + return + } + + rows, err := tx.Query(ctx, actorSchema.GetUpdatedAtHeightQuery(height)) + if err != nil { + return nil, err + } + defer rows.Close() + + var actor types.BaseActor + for rows.Next() { + if err = rows.Scan( + &actor.Address, + &actor.PublicKey, + &actor.StakedTokens, + &actor.ActorSpecificParam, + &actor.OutputAddress, + &actor.PausedHeight, + &actor.UnstakingHeight, + &actor.Chains, + &height, + ); err != nil { + return + } + + if actorSchema.GetChainsTableName() == "" { + continue + } + + chainRows, chainsErr := tx.Query(ctx, actorSchema.GetChainsQuery(actor.Address, height)) + if err != nil { + return nil, chainsErr // Why couldn't I just `return` here and use `err`? + } + defer chainRows.Close() + + var chainAddr string + var chainID string + var chainEndHeight int64 // unused + for rows.Next() { + err = rows.Scan(&chainAddr, &chainID, &chainEndHeight) + if err != nil { + return + } + if chainAddr != actor.Address { + return nil, fmt.Errorf("weird") + } + actor.Chains = append(actor.Chains, chainID) + } + + actors = append(actors, actor) + } + + return +} + func (p *PostgresContext) GetActor(actorSchema types.ProtocolActorSchema, address []byte, height int64) (actor types.BaseActor, err error) { ctx, txn, err := p.DB.GetCtxAndTxn() if err != nil { diff --git a/persistence/state.go b/persistence/state.go new file mode 100644 index 000000000..6e7abd9e7 --- /dev/null +++ b/persistence/state.go @@ -0,0 +1,103 @@ +package persistence + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "log" + "sort" + + "github.com/celestiaorg/smt" + typesUtil "github.com/pokt-network/pocket/utility/types" + "google.golang.org/protobuf/proto" +) + +type MerkleTree float64 + +// A work-in-progress list of all the trees we need to update to maintain the overall state +const ( + // Actors + AppMerkleTree MerkleTree = iota + ValMerkleTree + FishMerkleTree + ServiceNodeMerkleTree + AccountMerkleTree + PoolMerkleTree + // Data / state + BlocksMerkleTree + ParamsMerkleTree + FlagsMerkleTree + lastMerkleTree // Used for iteration purposes only - see https://stackoverflow.com/a/64178235/768439 +) + +func newMerkleTrees() (map[MerkleTree]*smt.SparseMerkleTree, error) { + // We need a separate Merkle tree for each type of actor or storage + trees := make(map[MerkleTree]*smt.SparseMerkleTree, int(lastMerkleTree)) + + for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { + // TODO_IN_THIS_COMMIT: Rather than using `NewSimpleMap`, use a disk based key-value store + nodeStore := smt.NewSimpleMap() + valueStore := smt.NewSimpleMap() + + trees[treeType] = smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New()) + } + return trees, nil +} + +func loadMerkleTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { + log.Fatalf("loadMerkleTrees not implemented yet") +} + +// DISCUSS_IN_THIS_COMMIT(drewskey): Thoughts on this approach? +// 1. Retrieves all of the actors / data types updated at the current height +// 2. Updates the Merkle Tree associated with each actor / data type +// - This operation is idempotent so you can call `updateStateHash` as often as you want +// 3. Update the context's "cached" state hash +// 4. Returns the state hash +func (p *PostgresContext) updateStateHash() ([]byte, error) { + // Update all the merkle trees + for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { + switch treeType { + case AppMerkleTree: + apps, err := p.getAppsUpdated(p.Height) + if err != nil { + return nil, typesUtil.NewError(typesUtil.Code(42), "Couldn't figure out apps updated") // TODO_IN_THIS_COMMIT + } + for _, app := range apps { + appBytes, err := proto.Marshal(app) + if err != nil { + return nil, err + } + // An update results in a create/update that is idempotent + addrBz, err := hex.DecodeString(app.Address) + if err != nil { + return nil, err + } + if _, err := p.MerkleTrees[treeType].Update(addrBz, appBytes); err != nil { + return nil, err + } + // TODO_IN_THIS_COMMIT: Add support for `Delete` operations to remove it from the tree + } + default: + log.Fatalln("Not handled yet in state commitment update", treeType) + } + } + + // Get the root of each Merkle Tree + roots := make([][]byte, 0) + for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { + roots = append(roots, p.MerkleTrees[treeType].Root()) + } + + // Sort the merkle roots lexicographically + sort.Slice(roots, func(r1, r2 int) bool { + return bytes.Compare(roots[r1], roots[r2]) < 0 + }) + + // Get the state hash + rootsConcat := bytes.Join(roots, []byte{}) + stateHash := sha256.Sum256(rootsConcat) + + p.StateHash = stateHash[:] + return p.StateHash, nil +} diff --git a/persistence/state_test.go b/persistence/state_test.go new file mode 100644 index 000000000..9570f4c00 --- /dev/null +++ b/persistence/state_test.go @@ -0,0 +1,15 @@ +package persistence + +import "testing" + +func TestStateHash_InitializeTrees(t *testing.T) { + +} + +func TestStateHash_LoadTrees(t *testing.T) { + +} + +func TestStateHash_ComputeStateHash(t *testing.T) { + +} diff --git a/persistence/types/base_actor.go b/persistence/types/base_actor.go index 9514b15eb..9cead8390 100644 --- a/persistence/types/base_actor.go +++ b/persistence/types/base_actor.go @@ -60,6 +60,10 @@ func (actor *BaseProtocolActorSchema) GetChainsTableSchema() string { return ProtocolActorChainsTableSchema(actor.chainsHeightConstraintName) } +func (actor *BaseProtocolActorSchema) GetUpdatedAtHeightQuery(height int64) string { + return SelectAtHeight(AllColsSelector, height, actor.tableName) +} + func (actor *BaseProtocolActorSchema) GetQuery(address string, height int64) string { return Select(AllColsSelector, address, height, actor.tableName) } diff --git a/persistence/types/protocol_actor.go b/persistence/types/protocol_actor.go index 4b57e8e40..9dd894fca 100644 --- a/persistence/types/protocol_actor.go +++ b/persistence/types/protocol_actor.go @@ -15,6 +15,9 @@ type ProtocolActorSchema interface { GetActorSpecificColName() string /*** Read/Get Queries ***/ + + // Returns a query to retrieve all of a Actors updated at that specific height. + GetUpdatedAtHeightQuery(height int64) string // Returns a query to retrieve all of a single Actor's attributes. GetQuery(address string, height int64) string // Returns all actors at that height diff --git a/persistence/types/shared_sql.go b/persistence/types/shared_sql.go index 5502c03d3..6c053b6de 100644 --- a/persistence/types/shared_sql.go +++ b/persistence/types/shared_sql.go @@ -69,6 +69,11 @@ func ProtocolActorChainsTableSchema(constraintName string) string { )`, AddressCol, ChainIDCol, HeightCol, DefaultBigInt, constraintName, AddressCol, ChainIDCol, HeightCol) } +func SelectAtHeight(selector string, height int64, tableName string) string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE height=%d`, + selector, tableName, height) +} + func Select(selector, address string, height int64, tableName string) string { return fmt.Sprintf(`SELECT %s FROM %s WHERE address='%s' AND height<=%d ORDER BY height DESC LIMIT 1`, selector, tableName, address, height) diff --git a/shared/indexer/indexer.go b/shared/indexer/indexer.go index 559ca1f6e..3c5d7167a 100644 --- a/shared/indexer/indexer.go +++ b/shared/indexer/indexer.go @@ -5,6 +5,7 @@ package indexer import ( "encoding/hex" "fmt" + "github.com/pokt-network/pocket/shared/codec" "github.com/jordanorelli/lexnum" @@ -111,7 +112,7 @@ type txIndexer struct { } func NewTxIndexer(databasePath string) (TxIndexer, error) { - db, err := kvstore.NewKVStore(databasePath) + db, err := kvstore.OpenKVStore(databasePath) return &txIndexer{ db: db, }, err @@ -198,22 +199,22 @@ func (indexer *txIndexer) get(key []byte) (TxResult, error) { func (indexer *txIndexer) indexByHash(hash, bz []byte) (hashKey []byte, err error) { key := indexer.hashKey(hash) - return key, indexer.db.Put(key, bz) + return key, indexer.db.Set(key, bz) } func (indexer *txIndexer) indexByHeightAndIndex(height int64, index int32, bz []byte) error { - return indexer.db.Put(indexer.heightAndIndexKey(height, index), bz) + return indexer.db.Set(indexer.heightAndIndexKey(height, index), bz) } func (indexer *txIndexer) indexBySender(sender string, bz []byte) error { - return indexer.db.Put(indexer.senderKey(sender), bz) + return indexer.db.Set(indexer.senderKey(sender), bz) } func (indexer *txIndexer) indexByRecipient(recipient string, bz []byte) error { if recipient == "" { return nil } - return indexer.db.Put(indexer.recipientKey(recipient), bz) + return indexer.db.Set(indexer.recipientKey(recipient), bz) } // key helper functions diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 91577b377..fdb0c0875 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -51,6 +51,7 @@ type PersistenceWriteContext interface { Commit() error Release() error + UpdateAppHash() ([]byte, error) AppHash() ([]byte, error) // Block Operations @@ -85,6 +86,10 @@ type PersistenceWriteContext interface { SetAppUnstakingHeightAndStatus(address []byte, unstakingHeight int64, status int) error SetAppStatusAndUnstakingHeightIfPausedBefore(pausedBeforeHeight, unstakingHeight int64, status int) error SetAppPauseHeight(address []byte, height int64) error + GetAppOutputAddress(operator []byte, height int64) (output []byte, err error) + // App Operations - For Tree Merkling + // GetAppsUpdated(height int64) ([][]byte, error) // Returns the apps updates at the given height + UpdateAppTree([][]byte) error // ServiceNode Operations InsertServiceNode(address []byte, publicKey []byte, output []byte, paused bool, status int, serviceURL string, stakedTokens string, chains []string, pausedHeight int64, unstakingHeight int64) error diff --git a/shared/types/genesis/validator.go b/shared/types/genesis/validator.go new file mode 100644 index 000000000..17c2af5fd --- /dev/null +++ b/shared/types/genesis/validator.go @@ -0,0 +1,49 @@ +package genesis + +// import ( +// "encoding/hex" +// "encoding/json" + +// "google.golang.org/protobuf/encoding/protojson" +// ) + +// // TODO_IN_THIS_COMMIT: See https://github.com/pokt-network/pocket/pull/139/files to remove this shit + +// // HACK: Since the protocol actor protobufs (e.g. validator, fisherman, etc) use `bytes` for some +// // fields (e.g. `address`, `output`, `publicKey`), we need to use a helper struct to unmarshal the +// // the types when they are defined via json (e.g. genesis file, testing configurations, etc...). +// // Alternative solutions could include whole wrapper structs (i.e. duplication of schema definition), +// // using strings instead of bytes (i.e. major change with downstream effects) or avoid defining these +// // types in json altogether (i.e. limitation of usability). +// type JsonBytesLoaderHelper struct { +// Address HexData `json:"address,omitempty"` +// PublicKey HexData `json:"public_key,omitempty"` +// Output HexData `json:"output,omitempty"` +// } + +// type HexData []byte + +// func (h *HexData) UnmarshalJSON(data []byte) error { +// var s string +// if err := json.Unmarshal(data, &s); err != nil { +// return err +// } +// decoded, err := hex.DecodeString(s) +// if err != nil { +// return err +// } +// *h = HexData(decoded) +// return nil +// } + +// func (v *Validator) UnmarshalJSON(data []byte) error { +// var jh JsonBytesLoaderHelper +// json.Unmarshal(data, &jh) + +// protojson.Unmarshal(data, v) +// v.Address = jh.Address +// v.PublicKey = jh.PublicKey +// v.Output = jh.Output + +// return nil +// } diff --git a/utility/block.go b/utility/block.go index 6c09cd8cf..acf148ca4 100644 --- a/utility/block.go +++ b/utility/block.go @@ -89,6 +89,9 @@ func (u *UtilityContext) EndBlock(proposer []byte) typesUtil.Error { if err := u.BeginUnstakingMaxPaused(); err != nil { return err } + if _, err := u.Context.UpdateAppHash(); err != nil { + return typesUtil.ErrAppHash(err) + } return nil }