Partitioning. Operation hash to bytes. Some performance issues. (#978)

* Partitioning. Operation hash to bytes. Remove mumbainet

* Fix: saving big map diffs

* Fix: unuseful error notifications (#981)

* More informative errors on view execution (#979)

* Remove mumbainet (#980)

* Partitioning. Operation hash to bytes. Remove mumbainet

* Fix: saving big map diffs

* Fix: do not send execute view errors to sentry

* Fix: try to improve performance of last query

* Fix: for condition

* Build GUI

* Avoid too many queries of create partiotions

* Fix: opg query

* Fix: opg response

* Fix: OPG computations was moved to app from database

* Fix: opg by partitions

* Fix: last id filter

* Fix: OPG query

* Fix: OPG query

* Fix: OPG query

* Fix: OPG query

* Fix: OPG query

* Fix: OPG query

* Fix: upgrade GUI version

* Fix: get last_action for contract from database

* add timestamp filter to last query

* Fix: contract stats query

* Fix: operations.Last

* Fix: big map stats query

* Fix: contract stats endpoint

* Log queries

* Add log_queries key to config

* Fix: last operation query

* Fix: stats for empty big maps

* Fix: stats for empty big maps

* Fix: default map value

* Update frontend tag in sandbox

.github/workflows/build.yml

@@ -83,7 +83,7 @@ jobs:
           context: .
           file: build/sandbox/Dockerfile
           build-args: |
-            TAG=4.5.4
+            TAG=4.5.7
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/tests_master.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: '1.20'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
@@ -25,7 +25,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: checkout code
         uses: actions/checkout@v2
       - uses: actions/cache@v2

.github/workflows/tests_pr.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: '1.20'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
@@ -25,7 +25,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: checkout code
         uses: actions/checkout@v2
       - uses: actions/cache@v2

build/api/Dockerfile

@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.19-alpine as builder
+FROM golang:1.20-alpine as builder
 
 ENV CGO_ENABLED=0
 ENV GO111MODULE=on

build/indexer/Dockerfile

@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.19-alpine as builder
+FROM golang:1.20-alpine as builder
 
 ENV CGO_ENABLED=0
 ENV GO111MODULE=on

build/sandbox/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14 AS build
+FROM node:16 AS build
 
 ARG TAG
 RUN git clone --depth=1 --branch ${TAG} https://github.com/baking-bad/bcd.git /bcd

cmd/api/handlers/entrypoints.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/baking-bad/bcdhub/internal/bcd"
 	"github.com/baking-bad/bcdhub/internal/bcd/ast"
+	"github.com/baking-bad/bcdhub/internal/bcd/encoding"
 	"github.com/baking-bad/bcdhub/internal/bcd/formatter"
 	"github.com/baking-bad/bcdhub/internal/bcd/types"
 	"github.com/baking-bad/bcdhub/internal/config"
@@ -151,6 +152,18 @@ func GetEntrypointSchema() gin.HandlerFunc {
 			return
 		}
 
+		var (
+			hash []byte
+			err  error
+		)
+
+		if esReq.Hash != "" {
+			hash, err = encoding.DecodeBase58(esReq.Hash)
+			if handleError(c, ctx.Storage, err, http.StatusBadRequest) {
+				return
+			}
+		}
+
 		symLink, err := getCurrentSymLink(ctx.Blocks)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
@@ -207,7 +220,7 @@ func GetEntrypointSchema() gin.HandlerFunc {
 					return
 				}
 
-				opg, err := ctx.Operations.GetByHashAndCounter(esReq.Hash, int64(*esReq.Counter))
+				opg, err := ctx.Operations.GetByHashAndCounter(hash, int64(*esReq.Counter))
 				if handleError(c, ctx.Storage, err, 0) {
 					return
 				}

cmd/api/handlers/error.go

@@ -7,13 +7,18 @@ import (
 	"github.com/baking-bad/bcdhub/internal/bcd/ast"
 	"github.com/baking-bad/bcdhub/internal/logger"
 	"github.com/baking-bad/bcdhub/internal/models"
+	"github.com/baking-bad/bcdhub/internal/noderpc"
 	sentrygin "github.com/getsentry/sentry-go/gin"
 	"github.com/gin-gonic/gin"
 	jsoniter "github.com/json-iterator/go"
 )
 
 var json = jsoniter.ConfigCompatibleWithStandardLibrary
 
+func skipError(err error) bool {
+	return errors.Is(err, noderpc.ErrNodeRPCError)
+}
+
 func handleError(c *gin.Context, repo models.GeneralRepository, err error, code int) bool {
 	if err == nil {
 		return false
@@ -24,12 +29,13 @@ func handleError(c *gin.Context, repo models.GeneralRepository, err error, code
 		err = errors.New("invalid authentication")
 	case 0:
 		code = getErrorCode(err, repo)
-		if code == http.StatusInternalServerError {
+		if code == http.StatusInternalServerError && !skipError(err) {
 			if hub := sentrygin.GetHubFromContext(c); hub != nil {
 				hub.CaptureMessage(err.Error())
 			}
-			logger.Err(err)
 		}
+
+		logger.Err(err)
 	}
 
 	c.AbortWithStatusJSON(code, getErrorMessage(err, repo))

cmd/api/handlers/migrations.go

@@ -3,6 +3,7 @@ package handlers
 import (
 	"net/http"
 
+	"github.com/baking-bad/bcdhub/internal/bcd/encoding"
 	"github.com/baking-bad/bcdhub/internal/config"
 	"github.com/baking-bad/bcdhub/internal/models/migration"
 	"github.com/gin-gonic/gin"
@@ -61,10 +62,14 @@ func prepareMigrations(ctx *config.Context, data []migration.Migration) ([]Migra
 		if err != nil && !ctx.Storage.IsRecordNotFound(err) {
 			return nil, err
 		}
+		var hash string
+		if len(data[i].Hash) > 0 {
+			hash = encoding.MustEncodeOperationHash(data[i].Hash)
+		}
 		result[i] = Migration{
 			Level:        data[i].Level,
 			Timestamp:    data[i].Timestamp,
-			Hash:         data[i].Hash,
+			Hash:         hash,
 			Protocol:     proto.Hash,
 			PrevProtocol: prevProto.Hash,
 			Kind:         data[i].Kind.String(),

cmd/api/handlers/operations.go

@@ -7,6 +7,7 @@ import (
 	"github.com/baking-bad/bcdhub/internal/bcd"
 	"github.com/baking-bad/bcdhub/internal/bcd/ast"
 	"github.com/baking-bad/bcdhub/internal/bcd/consts"
+	"github.com/baking-bad/bcdhub/internal/bcd/encoding"
 	"github.com/baking-bad/bcdhub/internal/bcd/formatter"
 	formattererror "github.com/baking-bad/bcdhub/internal/bcd/formatter/error"
 	"github.com/baking-bad/bcdhub/internal/bcd/tezerrors"
@@ -17,6 +18,7 @@ import (
 	"github.com/baking-bad/bcdhub/internal/models/operation"
 	modelTypes "github.com/baking-bad/bcdhub/internal/models/types"
 	"github.com/baking-bad/bcdhub/internal/parsers/storage"
+	"github.com/baking-bad/bcdhub/internal/postgres/core"
 	"github.com/gin-gonic/gin"
 	"github.com/pkg/errors"
 	"github.com/tidwall/gjson"
@@ -113,9 +115,14 @@ func GetOperation() gin.HandlerFunc {
 		operations := make([]operation.Operation, 0)
 		var foundContext *config.Context
 
+		hash, err := encoding.DecodeBase58(req.Hash)
+		if handleError(c, any.Storage, err, http.StatusBadRequest) {
+			return
+		}
+
 		network := modelTypes.NewNetwork(queryReq.Network)
 		if ctx, ok := ctxs[network]; ok {
-			op, err := ctx.Operations.GetByHash(req.Hash)
+			op, err := ctx.Operations.GetByHash(hash)
 			if err != nil {
 				if !ctx.Storage.IsRecordNotFound(err) {
 					handleError(c, ctx.Storage, err, 0)
@@ -127,7 +134,7 @@ func GetOperation() gin.HandlerFunc {
 			}
 		} else {
 			for _, ctx := range ctxs {
-				op, err := ctx.Operations.GetByHash(req.Hash)
+				op, err := ctx.Operations.GetByHash(hash)
 				if err != nil {
 					if !ctx.Storage.IsRecordNotFound(err) {
 						handleError(c, ctx.Storage, err, 0)
@@ -383,19 +390,24 @@ func GetByHashAndCounter() gin.HandlerFunc {
 			return
 		}
 
+		hash, err := encoding.DecodeBase58(req.Hash)
+		if handleError(c, ctxs.Any().Storage, err, http.StatusBadRequest) {
+			return
+		}
+
 		var opg []operation.Operation
 		var foundContext *config.Context
 
 		ctx, err := ctxs.Get(modelTypes.NewNetwork(args.Network))
 		if err == nil {
-			opg, err = ctx.Operations.GetByHashAndCounter(req.Hash, req.Counter)
+			opg, err = ctx.Operations.GetByHashAndCounter(hash, req.Counter)
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
 			foundContext = ctx
 		} else {
 			for _, ctx := range ctxs {
-				opg, err = ctx.Operations.GetByHashAndCounter(req.Hash, req.Counter)
+				opg, err = ctx.Operations.GetByHashAndCounter(hash, req.Counter)
 				if handleError(c, ctx.Storage, err, 0) {
 					return
 				}
@@ -608,6 +620,9 @@ func getStorageDiff(ctx *config.Context, destinationID int64, bmd []bigmapdiff.B
 		map[string]interface{}{
 			"destination_id": destinationID,
 			"status":         modelTypes.OperationStatusApplied,
+			"timestamp": core.TimestampFilter{
+				Lt: op.Timestamp,
+			},
 		}, op.ID)
 	if err == nil {
 		prevStorage = &ast.TypedAst{

cmd/api/handlers/responses.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/baking-bad/bcdhub/internal/bcd/ast"
+	"github.com/baking-bad/bcdhub/internal/bcd/encoding"
 	"github.com/baking-bad/bcdhub/internal/bcd/formatter"
 	"github.com/baking-bad/bcdhub/internal/bcd/tezerrors"
 	"github.com/baking-bad/bcdhub/internal/models/account"
@@ -67,7 +68,9 @@ type Operation struct {
 // FromModel -
 func (o *Operation) FromModel(operation operation.Operation) {
 	o.ID = operation.ID
-	o.Hash = operation.Hash
+	if len(operation.Hash) > 0 {
+		o.Hash = encoding.MustEncodeOperationHash(operation.Hash)
+	}
 	o.Internal = operation.Internal
 	o.Timestamp = operation.Timestamp.UTC()
 
@@ -97,9 +100,13 @@ func (o *Operation) FromModel(operation operation.Operation) {
 
 // ToModel -
 func (o *Operation) ToModel() operation.Operation {
+	var hash []byte
+	if o.Hash != "" {
+		hash = encoding.MustDecodeBase58(o.Hash)
+	}
 	return operation.Operation{
 		ID:        o.ID,
-		Hash:      o.Hash,
+		Hash:      hash,
 		Internal:  o.Internal,
 		Timestamp: o.Timestamp,
 		Level:     o.Level,
@@ -604,6 +611,10 @@ type OPGResponse struct {
 
 // NewOPGResponse -
 func NewOPGResponse(opg operation.OPG) OPGResponse {
+	var hash string
+	if len(opg.Hash) > 0 {
+		hash = encoding.MustEncodeOperationHash(opg.Hash)
+	}
 	return OPGResponse{
 		LastID:       opg.LastID,
 		ContentIndex: opg.ContentIndex,
@@ -612,7 +623,7 @@ func NewOPGResponse(opg operation.OPG) OPGResponse {
 		TotalCost:    opg.TotalCost,
 		Flow:         opg.Flow,
 		Internals:    opg.Internals,
-		Hash:         opg.Hash,
+		Hash:         hash,
 		Entrypoint:   opg.Entrypoint,
 		Timestamp:    opg.Timestamp,
 		Status:       opg.Status.String(),
@@ -636,8 +647,13 @@ func NewEvent(o operation.Operation) (*Event, error) {
 		return nil, nil
 	}
 
+	var hash string
+	if len(o.Hash) > 0 {
+		hash = encoding.MustEncodeOperationHash(o.Hash)
+	}
+
 	e := &Event{
-		Hash:      o.Hash,
+		Hash:      hash,
 		Status:    o.Status.String(),
 		Timestamp: o.Timestamp,
 		Level:     o.Level,

cmd/api/handlers/run_code.go

@@ -94,7 +94,7 @@ func RunOperation() gin.HandlerFunc {
 		}
 		parser := operations.NewGroup(parserParams)
 
-		store := postgres.NewStore(ctx.StorageDB.DB)
+		store := postgres.NewStore(ctx.StorageDB.DB, ctx.Partitions)
 		if err := parser.Parse(response, store); handleError(c, ctx.Storage, err, 0) {
 			return
 		}
@@ -304,7 +304,7 @@ func parseBigMapDiffs(c *gin.Context, ctx *config.Context, response noderpc.RunC
 		},
 	}
 
-	store := postgres.NewStore(ctx.StorageDB.DB)
+	store := postgres.NewStore(ctx.StorageDB.DB, ctx.Partitions)
 	switch operation.Kind {
 	case types.OperationKindTransaction.String():
 		err = specific.StorageParser.ParseTransaction(nodeOperation, &model, store)

cmd/api/handlers/tickets.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/baking-bad/bcdhub/internal/bcd/ast"
+	"github.com/baking-bad/bcdhub/internal/bcd/encoding"
 	"github.com/baking-bad/bcdhub/internal/config"
 	"github.com/gin-gonic/gin"
 )
@@ -72,7 +73,9 @@ func GetContractTicketUpdates() gin.HandlerFunc {
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
-			update.OperationHash = operation.Hash
+			if len(operation.Hash) > 0 {
+				update.OperationHash = encoding.MustEncodeOperationHash(operation.Hash)
+			}
 
 			response = append(response, update)
 		}

cmd/api/handlers/views.go

@@ -2,7 +2,6 @@ package handlers
 
 import (
 	"context"
-	"errors"
 	"io"
 	"net/http"
 	"time"
@@ -15,6 +14,7 @@ import (
 	"github.com/baking-bad/bcdhub/internal/models/contract"
 	"github.com/baking-bad/bcdhub/internal/views"
 	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
 )
 
 var (

cmd/indexer/indexer/indexer.go

@@ -254,7 +254,7 @@ func (bi *BlockchainIndexer) handleBlock(ctx context.Context, block *Block) erro
 				}
 			}
 
-			store := postgres.NewStore(tx)
+			store := postgres.NewStore(tx, bi.Partitions)
 			if err := bi.implicitMigration(ctx, block, bi.currentProtocol, store); err != nil {
 				return err
 			}
@@ -263,7 +263,7 @@ func (bi *BlockchainIndexer) handleBlock(ctx context.Context, block *Block) erro
 				return err
 			}
 
-			if err := store.Save(); err != nil {
+			if err := store.Save(ctx); err != nil {
 				return err
 			}
 
@@ -543,7 +543,7 @@ func (bi *BlockchainIndexer) vestingMigration(ctx context.Context, head noderpc.
 		return err
 	}
 
-	store := postgres.NewStore(tx)
+	store := postgres.NewStore(tx, bi.Partitions)
 
 	for _, address := range addresses {
 		if !bcd.IsContract(address) {
@@ -560,7 +560,7 @@ func (bi *BlockchainIndexer) vestingMigration(ctx context.Context, head noderpc.
 		}
 	}
 
-	return store.Save()
+	return store.Save(ctx)
 }
 
 func (bi *BlockchainIndexer) reinit(ctx context.Context, cfg config.Config, indexerConfig config.IndexerConfig) error {