From 8b532166181156f8526ab157ae083ab280116910 Mon Sep 17 00:00:00 2001 From: Artem Poltorzhitskiy Date: Tue, 30 Mar 2021 20:07:06 +0300 Subject: [PATCH] Feature: API page size in config (#574) --- cmd/api/handlers/context.go | 2 +- cmd/api/handlers/requests.go | 4 +-- cmd/api/handlers/tokens.go | 15 +++-------- cmd/api/main.go | 2 +- cmd/api/validations/validations.go | 26 +++++++++++++++++-- cmd/compiler/main.go | 2 +- cmd/indexer/indexer/boost.go | 2 +- cmd/metrics/main.go | 2 +- configs/development.yml | 3 ++- configs/production.yml | 4 ++- configs/sandbox.yml | 1 + configs/you.yml | 19 +++----------- docker-compose.gateway.yml | 10 +++++--- docker-compose.sandbox.yml | 6 ++--- internal/config/config.go | 31 +++++++++++++---------- internal/config/options.go | 4 +-- internal/elastic/bigmapdiff/data.go | 7 ++--- internal/elastic/bigmapdiff/storage.go | 7 ++--- internal/elastic/contract/storage.go | 6 +++-- internal/elastic/core/elastic.go | 15 ++++++++--- internal/elastic/core/events.go | 5 +--- internal/elastic/core/scroll.go | 12 +++++++++ internal/elastic/operation/storage.go | 9 +++---- internal/elastic/tokenbalance/storage.go | 4 +-- internal/elastic/tokenmetadata/storage.go | 1 + internal/parsers/tzip/tokens/parser.go | 2 +- scripts/api_tester/main.go | 2 +- scripts/esctl/main.go | 2 +- scripts/migration/main.go | 2 +- scripts/nginx/main.go | 2 +- 30 files changed, 115 insertions(+), 94 deletions(-) diff --git a/cmd/api/handlers/context.go b/cmd/api/handlers/context.go index 50010f708..3afb6a61e 100644 --- a/cmd/api/handlers/context.go +++ b/cmd/api/handlers/context.go @@ -29,7 +29,7 @@ func NewContext(cfg config.Config) (*Context, error) { } ctx := config.NewContext( - config.WithStorage(cfg.Storage), + config.WithStorage(cfg.Storage, int64(cfg.API.PageSize)), config.WithRPC(cfg.RPC), config.WithDatabase(cfg.DB), config.WithShare(cfg.SharePath), diff --git a/cmd/api/handlers/requests.go b/cmd/api/handlers/requests.go index ad2519db1..c0298294e 100644 --- a/cmd/api/handlers/requests.go +++ b/cmd/api/handlers/requests.go @@ -65,12 +65,12 @@ type operationsRequest struct { type pageableRequest struct { Offset int64 `form:"offset" binding:"min=0"` - Size int64 `form:"size" binding:"min=0,max=10"` + Size int64 `form:"size" binding:"min=0,bcd_max_size=10"` } type cursorRequest struct { LastID string `form:"last_id" binding:"omitempty,numeric"` - Size int64 `form:"size" binding:"min=0,max=10"` + Size int64 `form:"size" binding:"min=0,bcd_max_size=10"` } type searchRequest struct { diff --git a/cmd/api/handlers/tokens.go b/cmd/api/handlers/tokens.go index ae382d32e..2e3e46f5b 100644 --- a/cmd/api/handlers/tokens.go +++ b/cmd/api/handlers/tokens.go @@ -36,9 +36,7 @@ func (ctx *Context) GetFA(c *gin.Context) { if err := c.BindQuery(&cursorReq); ctx.handleError(c, err, http.StatusBadRequest) { return } - if cursorReq.Size == 0 { - cursorReq.Size = 20 - } + contracts, total, err := ctx.Contracts.GetTokens(req.Network, "", cursorReq.Offset, cursorReq.Size) if ctx.handleError(c, err, 0) { return @@ -78,9 +76,7 @@ func (ctx *Context) GetFAByVersion(c *gin.Context) { if err := c.BindQuery(&cursorReq); ctx.handleError(c, err, http.StatusBadRequest) { return } - if cursorReq.Size == 0 { - cursorReq.Size = 20 - } + if req.Version == "fa12" { req.Version = consts.FA12Tag } @@ -303,9 +299,6 @@ func (ctx *Context) GetContractTokens(c *gin.Context) { pageReq.TokenID = new(int64) *pageReq.TokenID = -1 } - if pageReq.Size == 0 { - pageReq.Size = 10 - } metadata, err := ctx.TokenMetadata.Get([]tokenmetadata.GetContext{{ Contract: req.Address, @@ -452,9 +445,7 @@ func (ctx *Context) GetTokenMetadata(c *gin.Context) { queryParams.TokenID = new(int64) *queryParams.TokenID = -1 } - if queryParams.Size == 0 { - queryParams.Size = 10 - } + tokens, err := ctx.getTokensWithSupply(tokenmetadata.GetContext{ Contract: queryParams.Contract, Network: req.Network, diff --git a/cmd/api/main.go b/cmd/api/main.go index 0eb365db1..030933412 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -65,7 +65,7 @@ func (api *app) makeRouter() { r.MaxMultipartMemory = 4 << 20 // max upload size 4 MiB if v, ok := binding.Validator.Engine().(*validator.Validate); ok { - if err := validations.Register(v, api.Context.Config.API.Networks); err != nil { + if err := validations.Register(v, api.Context.Config.API); err != nil { logger.Fatal(err) } } diff --git a/cmd/api/validations/validations.go b/cmd/api/validations/validations.go index c226cfed1..173a4a1af 100644 --- a/cmd/api/validations/validations.go +++ b/cmd/api/validations/validations.go @@ -6,13 +6,18 @@ import ( "github.com/baking-bad/bcdhub/internal/bcd/consts" "github.com/baking-bad/bcdhub/internal/compiler/compilation" + "github.com/baking-bad/bcdhub/internal/config" "github.com/baking-bad/bcdhub/internal/helpers" "github.com/btcsuite/btcutil/base58" "gopkg.in/go-playground/validator.v9" ) +const ( + defaultMaxSize = 10 +) + // Register - -func Register(v *validator.Validate, networks []string) error { +func Register(v *validator.Validate, cfg config.APIConfig) error { if err := v.RegisterValidation("address", addressValidator()); err != nil { return err } @@ -21,7 +26,7 @@ func Register(v *validator.Validate, networks []string) error { return err } - if err := v.RegisterValidation("network", networkValidator(networks)); err != nil { + if err := v.RegisterValidation("network", networkValidator(cfg.Networks)); err != nil { return err } @@ -49,6 +54,10 @@ func Register(v *validator.Validate, networks []string) error { return err } + if err := v.RegisterValidation("bcd_max_size", maxSizeValidator(cfg.PageSize)); err != nil { + return err + } + return nil } @@ -136,6 +145,19 @@ func searchStringValidator() validator.Func { } } +func maxSizeValidator(maxSize uint64) validator.Func { + return func(fl validator.FieldLevel) bool { + if maxSize == 0 { + maxSize = defaultMaxSize + } + fieldValue := fl.Field().Int() + if fieldValue < 0 { + return false + } + return uint64(fieldValue) <= maxSize + } +} + func greatThanInt64PtrValidator() validator.Func { return func(fl validator.FieldLevel) bool { field := fl.Field() diff --git a/cmd/compiler/main.go b/cmd/compiler/main.go index 5e258a6e8..a63e1e9af 100644 --- a/cmd/compiler/main.go +++ b/cmd/compiler/main.go @@ -42,7 +42,7 @@ func main() { config.WithRPC(cfg.RPC), config.WithDatabase(cfg.DB), config.WithRabbit(cfg.RabbitMQ, cfg.Compiler.ProjectName, cfg.Compiler.MQ), - config.WithStorage(cfg.Storage), + config.WithStorage(cfg.Storage, 0), config.WithAWS(cfg.Compiler.AWS), ), } diff --git a/cmd/indexer/indexer/boost.go b/cmd/indexer/indexer/boost.go index f9b5b132b..aff3e0147 100644 --- a/cmd/indexer/indexer/boost.go +++ b/cmd/indexer/indexer/boost.go @@ -132,7 +132,7 @@ func (bi *BoostIndexer) fetchExternalProtocols() error { // NewBoostIndexer - func NewBoostIndexer(cfg config.Config, network string, opts ...BoostIndexerOption) (*BoostIndexer, error) { logger.WithNetwork(network).Info("Creating indexer object...") - es := core.WaitNew(cfg.Storage.URI, cfg.Storage.Timeout) + es := core.WaitNew(cfg.Storage.URI, cfg.Storage.Timeout, 0) rpcProvider, ok := cfg.RPC[network] if !ok { diff --git a/cmd/metrics/main.go b/cmd/metrics/main.go index 04f0a4c09..605e7e96c 100644 --- a/cmd/metrics/main.go +++ b/cmd/metrics/main.go @@ -89,7 +89,7 @@ func main() { } configCtx := config.NewContext( - config.WithStorage(cfg.Storage), + config.WithStorage(cfg.Storage, 0), config.WithRPC(cfg.RPC), config.WithDatabase(cfg.DB), config.WithRabbit(cfg.RabbitMQ, cfg.Metrics.ProjectName, cfg.Metrics.MQ), diff --git a/configs/development.yml b/configs/development.yml index 734bf1bb7..ea2fbf9fc 100644 --- a/configs/development.yml +++ b/configs/development.yml @@ -59,7 +59,7 @@ share_path: ${HOME}/.bcd base_url: http://localhost:8080 ipfs: - - https://ipfs.baking-bad.org + - https://cloudflare-ipfs.com - https://ipfs.io - https://dweb.link @@ -74,6 +74,7 @@ api: oauth_enabled: true sentry_enabled: false seed_enabled: false + page_size: ${PAGE_SIZE:-10} frontend: ga_enabled: false mempool_enabled: true diff --git a/configs/production.yml b/configs/production.yml index 7d788c0e4..96937a600 100644 --- a/configs/production.yml +++ b/configs/production.yml @@ -60,7 +60,8 @@ share_path: /etc/bcd base_url: https://better-call.dev ipfs: - - https://ipfs.baking-bad.org + - ${SANDBOX_IPFS_GATEWAY:-https://ipfs.baking-bad.org} + - https://cloudflare-ipfs.com - https://ipfs.io - https://dweb.link @@ -75,6 +76,7 @@ api: oauth_enabled: true sentry_enabled: true seed_enabled: false + page_size: ${PAGE_SIZE:-10} frontend: ga_enabled: true mempool_enabled: true diff --git a/configs/sandbox.yml b/configs/sandbox.yml index d43b2fee8..f80b6af46 100644 --- a/configs/sandbox.yml +++ b/configs/sandbox.yml @@ -31,6 +31,7 @@ api: oauth_enabled: false sentry_enabled: false seed_enabled: true + page_size: ${PAGE_SIZE:-10} frontend: ga_enabled: false mempool_enabled: false diff --git a/configs/you.yml b/configs/you.yml index 338d2c079..ef66e0920 100644 --- a/configs/you.yml +++ b/configs/you.yml @@ -1,6 +1,6 @@ rpc: mainnet: - uri: https://rpc.tzkt.io/mainnet + uri: https://mainnet-tezos.giganode.io timeout: 20 edo2net: uri: https://rpc.tzkt.io/edo2net @@ -9,18 +9,6 @@ rpc: uri: https://rpc.tzkt.io/florencenobanet timeout: 20 -tzkt: - mainnet: - uri: https://api.tzkt.io/v1/ - services_uri: https://services.tzkt.io/v1/ - base_uri: https://tzkt.io/ - timeout: 20 - edo2net: - uri: https://api.edo2net.tzkt.io/v1/ - services_uri: https://services.tzkt.io/edo2net/v1/ - base_uri: https://edo2net.tzkt.io/ - timeout: 20 - storage: uri: - http://${ELASTIC_HOSTNAME:-elastic}:9200 @@ -59,7 +47,8 @@ share_path: /etc/bcd base_url: https://you.better-call.dev ipfs: - - https://ipfs.baking-bad.org + - ${SANDBOX_IPFS_GATEWAY:-https://ipfs.baking-bad.org} + - https://cloudflare-ipfs.com - https://ipfs.io - https://dweb.link @@ -74,6 +63,7 @@ api: oauth_enabled: true sentry_enabled: true seed_enabled: false + page_size: ${PAGE_SIZE:-10} frontend: ga_enabled: false mempool_enabled: true @@ -108,7 +98,6 @@ indexer: publisher: true networks: mainnet: - boost: tzkt edo2net: florencenet: diff --git a/docker-compose.gateway.yml b/docker-compose.gateway.yml index 3607ea93e..c68e8d4f5 100644 --- a/docker-compose.gateway.yml +++ b/docker-compose.gateway.yml @@ -45,13 +45,15 @@ services: context: . dockerfile: build/api/Dockerfile environment: - - BCD_ENV=production + - BCD_ENV=you - GIN_MODE=release - POSTGRES_USER=root - POSTGRES_PASSWORD=root - POSTGRES_DB=bcd - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest + - PAGE_SIZE=100 + - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com depends_on: - elastic - db @@ -69,12 +71,13 @@ services: context: . dockerfile: build/indexer/Dockerfile environment: - - BCD_ENV=production + - BCD_ENV=you - POSTGRES_USER=root - POSTGRES_PASSWORD=root - POSTGRES_DB=bcd - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest + - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com depends_on: - elastic - mq @@ -90,12 +93,13 @@ services: context: . dockerfile: build/metrics/Dockerfile environment: - - BCD_ENV=production + - BCD_ENV=you - POSTGRES_USER=root - POSTGRES_PASSWORD=root - POSTGRES_DB=bcd - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest + - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com depends_on: - elastic - mq diff --git a/docker-compose.sandbox.yml b/docker-compose.sandbox.yml index 48734c6a9..2d9d7e28c 100644 --- a/docker-compose.sandbox.yml +++ b/docker-compose.sandbox.yml @@ -47,7 +47,7 @@ services: - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest - SANDBOX_NODE_URI=http://sandbox:8732 - - SANDBOX_IPFS_GATEWAY=https://ipfs.io + - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com depends_on: - elastic - db @@ -68,7 +68,7 @@ services: - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest - SANDBOX_NODE_URI=http://sandbox:8732 - - SANDBOX_IPFS_GATEWAY=https://ipfs.io + - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com depends_on: - elastic - mq @@ -90,7 +90,7 @@ services: - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest - SANDBOX_NODE_URI=http://sandbox:8732 - - SANDBOX_IPFS_GATEWAY=https://ipfs.io + - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com depends_on: - elastic - mq diff --git a/internal/config/config.go b/internal/config/config.go index 103f2a62b..8e5af1a5b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,20 +32,7 @@ type Config struct { IPFSGateways []string `yaml:"ipfs"` Domains TezosDomainsConfig `yaml:"domains"` - API struct { - ProjectName string `yaml:"project_name"` - Bind string `yaml:"bind"` - SwaggerHost string `yaml:"swagger_host"` - CorsEnabled bool `yaml:"cors_enabled"` - OAuthEnabled bool `yaml:"oauth_enabled"` - SentryEnabled bool `yaml:"sentry_enabled"` - SeedEnabled bool `yaml:"seed_enabled"` - Frontend FrontendConfig `yaml:"frontend"` - Seed SeedConfig `yaml:"seed"` - Networks []string `yaml:"networks"` - MQ MQConfig `yaml:"mq"` - Pinata PinataConfig `yaml:"pinata"` - } `yaml:"api"` + API APIConfig `yaml:"api"` Compiler struct { ProjectName string `yaml:"project_name"` @@ -171,6 +158,22 @@ type SeedConfig struct { } `yaml:"accounts"` } +type APIConfig struct { + ProjectName string `yaml:"project_name"` + Bind string `yaml:"bind"` + SwaggerHost string `yaml:"swagger_host"` + CorsEnabled bool `yaml:"cors_enabled"` + OAuthEnabled bool `yaml:"oauth_enabled"` + SentryEnabled bool `yaml:"sentry_enabled"` + SeedEnabled bool `yaml:"seed_enabled"` + Frontend FrontendConfig `yaml:"frontend"` + Seed SeedConfig `yaml:"seed"` + Networks []string `yaml:"networks"` + MQ MQConfig `yaml:"mq"` + Pinata PinataConfig `yaml:"pinata"` + PageSize uint64 `yaml:"page_size"` +} + // SentryConfig - type SentryConfig struct { Environment string `yaml:"environment"` diff --git a/internal/config/options.go b/internal/config/options.go index 0e0ad0d32..aac65d11c 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -64,7 +64,7 @@ func WithRPC(rpcConfig map[string]RPCConfig) ContextOption { } // WithStorage - -func WithStorage(cfg StorageConfig) ContextOption { +func WithStorage(cfg StorageConfig, maxPageSize int64) ContextOption { return func(ctx *Context) { if len(cfg.URI) == 0 { panic("Please set connection strings to storage in config") @@ -93,7 +93,7 @@ func WithStorage(cfg StorageConfig) ContextOption { panic(err) } } else { - es := core.WaitNew(cfg.URI, cfg.Timeout) + es := core.WaitNew(cfg.URI, cfg.Timeout, maxPageSize) ctx.Storage = es ctx.BigMapActions = bigmapaction.NewStorage(es) diff --git a/internal/elastic/bigmapdiff/data.go b/internal/elastic/bigmapdiff/data.go index 37ac29087..ebff9292c 100644 --- a/internal/elastic/bigmapdiff/data.go +++ b/internal/elastic/bigmapdiff/data.go @@ -3,7 +3,6 @@ package bigmapdiff import ( "fmt" - "github.com/baking-bad/bcdhub/internal/elastic/consts" "github.com/baking-bad/bcdhub/internal/elastic/core" "github.com/baking-bad/bcdhub/internal/models/bigmapdiff" ) @@ -27,7 +26,7 @@ type getBigMapDiffsCountResponse struct { } `json:"aggregations"` } -func buildGetContext(ctx *bigmapdiff.GetContext) core.Base { +func buildGetContext(ctx *bigmapdiff.GetContext, maxPageSize int64) core.Base { filters := make([]core.Item, 0) if ctx.Ptr != nil { @@ -41,9 +40,7 @@ func buildGetContext(ctx *bigmapdiff.GetContext) core.Base { filters = append(filters, core.QueryString(fmt.Sprintf("*%s*", ctx.Query), []string{"key", "key_hash", "key_strings", "bin_path", "value", "value_strings"})) } - if ctx.Size == 0 { - ctx.Size = consts.DefaultSize - } + ctx.Size = core.GetSize(ctx.Size, maxPageSize) if ctx.MaxLevel != nil { filters = append(filters, core.Range("level", core.Item{"lte": *ctx.MaxLevel})) diff --git a/internal/elastic/bigmapdiff/storage.go b/internal/elastic/bigmapdiff/storage.go index 275d2b411..74ece8873 100644 --- a/internal/elastic/bigmapdiff/storage.go +++ b/internal/elastic/bigmapdiff/storage.go @@ -3,7 +3,6 @@ package bigmapdiff import ( "encoding/json" - "github.com/baking-bad/bcdhub/internal/elastic/consts" "github.com/baking-bad/bcdhub/internal/elastic/core" "github.com/baking-bad/bcdhub/internal/models" "github.com/baking-bad/bcdhub/internal/models/bigmapdiff" @@ -283,9 +282,7 @@ func (storage *Storage) GetByPtrAndKeyHash(ptr int64, network, keyHash string, s ) b := core.Bool(mustQuery) - if size == 0 { - size = consts.DefaultSize - } + size = core.GetSize(size, storage.es.MaxPageSize) query := core.NewQuery().Query(b).Sort("level", "desc").Size(size).From(offset) @@ -374,7 +371,7 @@ func (storage *Storage) Get(ctx bigmapdiff.GetContext) ([]bigmapdiff.Bucket, err return nil, errors.Errorf("Invalid pointer value: %d", *ctx.Ptr) } - query := buildGetContext(&ctx) + query := buildGetContext(&ctx, storage.es.MaxPageSize) var response getBigMapDiffsWithKeysResponse if err := storage.es.Query([]string{models.DocBigMapDiff}, query, &response); err != nil { diff --git a/internal/elastic/contract/storage.go b/internal/elastic/contract/storage.go index 3b727c9e9..55edf8efe 100644 --- a/internal/elastic/contract/storage.go +++ b/internal/elastic/contract/storage.go @@ -298,7 +298,7 @@ func (storage *Storage) GetSameContracts(c contract.Contract, manager string, si return pcr, errors.Errorf("Invalid contract data") } - if size == 0 { + if size <= 0 { size = consts.DefaultSize } else if size+offset > core.MaxQuerySize { size = core.MaxQuerySize - offset @@ -364,7 +364,7 @@ func (storage *Storage) GetSimilarContracts(c contract.Contract, size, offset in return } - if size == 0 { + if size <= 0 { size = consts.DefaultSize } else if size+offset > core.MaxQuerySize { size = core.MaxQuerySize - offset @@ -508,6 +508,8 @@ func (storage *Storage) GetTokens(network, tokenInterface string, offset, size i ).Sort("timestamp", "desc") contracts := make([]contract.Contract, 0) + + size = core.GetSize(size, storage.es.MaxPageSize) ctx := core.NewScrollContext(storage.es, query, size, consts.DefaultScrollSize) ctx.Offset = offset if err := ctx.Get(&contracts); err != nil { diff --git a/internal/elastic/core/elastic.go b/internal/elastic/core/elastic.go index a171a6722..73ad25c5c 100644 --- a/internal/elastic/core/elastic.go +++ b/internal/elastic/core/elastic.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/baking-bad/bcdhub/internal/elastic/consts" "github.com/baking-bad/bcdhub/internal/logger" "github.com/baking-bad/bcdhub/internal/models" "github.com/cenkalti/backoff" @@ -22,10 +23,12 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary // Elastic - type Elastic struct { *elasticsearch.Client + + MaxPageSize int64 } // New - -func New(addresses []string) (*Elastic, error) { +func New(addresses []string, maxPageSize int64) (*Elastic, error) { retryBackoff := backoff.NewExponentialBackOff() elasticConfig := elasticsearch.Config{ Addresses: addresses, @@ -43,17 +46,21 @@ func New(addresses []string) (*Elastic, error) { return nil, err } - e := &Elastic{es} + if maxPageSize <= 0 { + maxPageSize = consts.DefaultSize + } + + e := &Elastic{es, maxPageSize} return e, e.TestConnection() } // WaitNew - -func WaitNew(addresses []string, timeout int) *Elastic { +func WaitNew(addresses []string, timeout int, maxPageSize int64) *Elastic { var es *Elastic var err error for es == nil { - es, err = New(addresses) + es, err = New(addresses, maxPageSize) if err != nil { logger.Warning("Waiting elastic up %d seconds...", timeout) time.Sleep(time.Second * time.Duration(timeout)) diff --git a/internal/elastic/core/events.go b/internal/elastic/core/events.go index 51ed053c0..2aafa61b9 100644 --- a/internal/elastic/core/events.go +++ b/internal/elastic/core/events.go @@ -5,7 +5,6 @@ import ( "github.com/baking-bad/bcdhub/internal/bcd" constants "github.com/baking-bad/bcdhub/internal/bcd/consts" - "github.com/baking-bad/bcdhub/internal/elastic/consts" "github.com/baking-bad/bcdhub/internal/models" "github.com/pkg/errors" ) @@ -16,9 +15,7 @@ func (e *Elastic) GetEvents(subscriptions []models.SubscriptionRequest, size, of return []models.Event{}, nil } - if size == 0 || size > 50 { // TODO: ??? - size = consts.DefaultSize - } + size = GetSize(size, e.MaxPageSize) shouldItems := make([]Item, 0) indicesMap := make(map[string]struct{}) diff --git a/internal/elastic/core/scroll.go b/internal/elastic/core/scroll.go index 4aa725711..93ad2385d 100644 --- a/internal/elastic/core/scroll.go +++ b/internal/elastic/core/scroll.go @@ -13,6 +13,18 @@ import ( const defaultScrollSize = 1000 +// GetSize - +func GetSize(size int64, defaultMaxSize int64) int64 { + switch { + case size <= 0: + return defaultMaxSize + case size > MaxQuerySize: + return MaxQuerySize + default: + return size + } +} + // ScrollContext - type ScrollContext struct { Query Base diff --git a/internal/elastic/operation/storage.go b/internal/elastic/operation/storage.go index a1b8b253e..769837106 100644 --- a/internal/elastic/operation/storage.go +++ b/internal/elastic/operation/storage.go @@ -24,11 +24,7 @@ func NewStorage(es *core.Elastic) *Storage { return &Storage{es} } -func (storage *Storage) getContractOPG(address, network string, size uint64, filters map[string]interface{}) ([]opgForContract, error) { - if size == 0 || size > core.MaxQuerySize { - size = consts.DefaultSize - } - +func (storage *Storage) getContractOPG(address, network string, size int64, filters map[string]interface{}) ([]opgForContract, error) { filtersString, err := prepareOperationFilters(filters) if err != nil { return nil, err @@ -82,7 +78,8 @@ func prepareOperationFilters(filters map[string]interface{}) (s string, err erro // GetByContract - func (storage *Storage) GetByContract(network, address string, size uint64, filters map[string]interface{}) (po operation.Pageable, err error) { - opg, err := storage.getContractOPG(address, network, size, filters) + limit := core.GetSize(int64(size), storage.es.MaxPageSize) + opg, err := storage.getContractOPG(address, network, limit, filters) if err != nil { return } diff --git a/internal/elastic/tokenbalance/storage.go b/internal/elastic/tokenbalance/storage.go index aa3441b67..e226ce563 100644 --- a/internal/elastic/tokenbalance/storage.go +++ b/internal/elastic/tokenbalance/storage.go @@ -103,9 +103,7 @@ func (storage *Storage) GetAccountBalances(network, address, contract string, si ), ).Sort("token_id", "desc").All() - if size == 0 { - size = consts.DefaultSize - } + size = core.GetSize(size, storage.es.MaxPageSize) tokenBalances := make([]tokenbalance.TokenBalance, 0) ctx := core.NewScrollContext(storage.es, query, size, consts.DefaultScrollSize) diff --git a/internal/elastic/tokenmetadata/storage.go b/internal/elastic/tokenmetadata/storage.go index 243172e52..e85595b48 100644 --- a/internal/elastic/tokenmetadata/storage.go +++ b/internal/elastic/tokenmetadata/storage.go @@ -21,6 +21,7 @@ func NewStorage(es *core.Elastic) *Storage { // Get - func (storage *Storage) Get(ctx []tokenmetadata.GetContext, size, offset int64) (tokens []tokenmetadata.TokenMetadata, err error) { query := buildGetTokenMetadataContext(ctx, true) + size = core.GetSize(size, storage.es.MaxPageSize) scrollCtx := core.NewScrollContext(storage.es, query, size, consts.DefaultScrollSize) scrollCtx.Offset = offset err = scrollCtx.Get(&tokens) diff --git a/internal/parsers/tzip/tokens/parser.go b/internal/parsers/tzip/tokens/parser.go index ffd5d6667..fca5eb7e6 100644 --- a/internal/parsers/tzip/tokens/parser.go +++ b/internal/parsers/tzip/tokens/parser.go @@ -100,7 +100,7 @@ func (t Parser) parse(address string, state block.Block) ([]tokenmetadata.TokenM bmd, err := t.bmdRepo.Get(bigmapdiff.GetContext{ Ptr: &ptr, Network: t.network, - Size: 1000, // TODO: max size + Size: 10000, CurrentLevel: &state.Level, Contract: address, }) diff --git a/scripts/api_tester/main.go b/scripts/api_tester/main.go index 1ea073a97..6991e711e 100644 --- a/scripts/api_tester/main.go +++ b/scripts/api_tester/main.go @@ -21,7 +21,7 @@ func main() { } ctx := config.NewContext( - config.WithStorage(cfg.Storage), + config.WithStorage(cfg.Storage, 0), config.WithRPC(cfg.RPC), config.WithShare(cfg.SharePath), config.WithTzKTServices(cfg.TzKT), diff --git a/scripts/esctl/main.go b/scripts/esctl/main.go index 8d453bf11..8331e4f66 100644 --- a/scripts/esctl/main.go +++ b/scripts/esctl/main.go @@ -30,7 +30,7 @@ func main() { } ctx = config.NewContext( - config.WithStorage(cfg.Storage), + config.WithStorage(cfg.Storage, 0), config.WithRabbit(cfg.RabbitMQ, "", cfg.Scripts.MQ), config.WithConfigCopy(cfg), config.WithRPC(cfg.RPC), diff --git a/scripts/migration/main.go b/scripts/migration/main.go index 085729adb..f4fcea90a 100644 --- a/scripts/migration/main.go +++ b/scripts/migration/main.go @@ -46,7 +46,7 @@ func main() { ctx := config.NewContext( config.WithShare(cfg.SharePath), - config.WithStorage(cfg.Storage), + config.WithStorage(cfg.Storage, 0), config.WithDatabase(cfg.DB), config.WithRPC(cfg.RPC), config.WithConfigCopy(cfg), diff --git a/scripts/nginx/main.go b/scripts/nginx/main.go index f393b3884..e34a05fc4 100644 --- a/scripts/nginx/main.go +++ b/scripts/nginx/main.go @@ -16,7 +16,7 @@ func main() { } ctx := config.NewContext( - config.WithStorage(cfg.Storage), + config.WithStorage(cfg.Storage, 0), config.WithConfigCopy(cfg), ) defer ctx.Close()