Feature: API page size in config (#574)

baking-bad · Mar 30, 2021 · 8b53216 · 8b53216
1 parent b4e1c11
commit 8b53216
Show file tree

Hide file tree

Showing 30 changed files with 115 additions and 94 deletions.
diff --git a/cmd/api/handlers/context.go b/cmd/api/handlers/context.go
@@ -29,7 +29,7 @@ func NewContext(cfg config.Config) (*Context, error) {
 	}
 
 	ctx := config.NewContext(
-		config.WithStorage(cfg.Storage),
+		config.WithStorage(cfg.Storage, int64(cfg.API.PageSize)),
 		config.WithRPC(cfg.RPC),
 		config.WithDatabase(cfg.DB),
 		config.WithShare(cfg.SharePath),

diff --git a/cmd/api/handlers/requests.go b/cmd/api/handlers/requests.go
@@ -65,12 +65,12 @@ type operationsRequest struct {
 
 type pageableRequest struct {
 	Offset int64 `form:"offset" binding:"min=0"`
-	Size   int64 `form:"size" binding:"min=0,max=10"`
+	Size   int64 `form:"size" binding:"min=0,bcd_max_size=10"`
 }
 
 type cursorRequest struct {
 	LastID string `form:"last_id" binding:"omitempty,numeric"`
-	Size   int64  `form:"size" binding:"min=0,max=10"`
+	Size   int64  `form:"size" binding:"min=0,bcd_max_size=10"`
 }
 
 type searchRequest struct {

diff --git a/cmd/api/handlers/tokens.go b/cmd/api/handlers/tokens.go
@@ -36,9 +36,7 @@ func (ctx *Context) GetFA(c *gin.Context) {
 	if err := c.BindQuery(&cursorReq); ctx.handleError(c, err, http.StatusBadRequest) {
 		return
 	}
-	if cursorReq.Size == 0 {
-		cursorReq.Size = 20
-	}
+
 	contracts, total, err := ctx.Contracts.GetTokens(req.Network, "", cursorReq.Offset, cursorReq.Size)
 	if ctx.handleError(c, err, 0) {
 		return
@@ -78,9 +76,7 @@ func (ctx *Context) GetFAByVersion(c *gin.Context) {
 	if err := c.BindQuery(&cursorReq); ctx.handleError(c, err, http.StatusBadRequest) {
 		return
 	}
-	if cursorReq.Size == 0 {
-		cursorReq.Size = 20
-	}
+
 	if req.Version == "fa12" {
 		req.Version = consts.FA12Tag
 	}
@@ -303,9 +299,6 @@ func (ctx *Context) GetContractTokens(c *gin.Context) {
 		pageReq.TokenID = new(int64)
 		*pageReq.TokenID = -1
 	}
-	if pageReq.Size == 0 {
-		pageReq.Size = 10
-	}
 
 	metadata, err := ctx.TokenMetadata.Get([]tokenmetadata.GetContext{{
 		Contract: req.Address,
@@ -452,9 +445,7 @@ func (ctx *Context) GetTokenMetadata(c *gin.Context) {
 		queryParams.TokenID = new(int64)
 		*queryParams.TokenID = -1
 	}
-	if queryParams.Size == 0 {
-		queryParams.Size = 10
-	}
+
 	tokens, err := ctx.getTokensWithSupply(tokenmetadata.GetContext{
 		Contract: queryParams.Contract,
 		Network:  req.Network,

diff --git a/cmd/api/main.go b/cmd/api/main.go
@@ -65,7 +65,7 @@ func (api *app) makeRouter() {
 	r.MaxMultipartMemory = 4 << 20 // max upload size 4 MiB
 
 	if v, ok := binding.Validator.Engine().(*validator.Validate); ok {
-		if err := validations.Register(v, api.Context.Config.API.Networks); err != nil {
+		if err := validations.Register(v, api.Context.Config.API); err != nil {
 			logger.Fatal(err)
 		}
 	}

diff --git a/cmd/api/validations/validations.go b/cmd/api/validations/validations.go
@@ -6,13 +6,18 @@ import (
 
 	"github.com/baking-bad/bcdhub/internal/bcd/consts"
 	"github.com/baking-bad/bcdhub/internal/compiler/compilation"
+	"github.com/baking-bad/bcdhub/internal/config"
 	"github.com/baking-bad/bcdhub/internal/helpers"
 	"github.com/btcsuite/btcutil/base58"
 	"gopkg.in/go-playground/validator.v9"
 )
 
+const (
+	defaultMaxSize = 10
+)
+
 // Register -
-func Register(v *validator.Validate, networks []string) error {
+func Register(v *validator.Validate, cfg config.APIConfig) error {
 	if err := v.RegisterValidation("address", addressValidator()); err != nil {
 		return err
 	}
@@ -21,7 +26,7 @@ func Register(v *validator.Validate, networks []string) error {
 		return err
 	}
 
-	if err := v.RegisterValidation("network", networkValidator(networks)); err != nil {
+	if err := v.RegisterValidation("network", networkValidator(cfg.Networks)); err != nil {
 		return err
 	}
 
@@ -49,6 +54,10 @@ func Register(v *validator.Validate, networks []string) error {
 		return err
 	}
 
+	if err := v.RegisterValidation("bcd_max_size", maxSizeValidator(cfg.PageSize)); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -136,6 +145,19 @@ func searchStringValidator() validator.Func {
 	}
 }
 
+func maxSizeValidator(maxSize uint64) validator.Func {
+	return func(fl validator.FieldLevel) bool {
+		if maxSize == 0 {
+			maxSize = defaultMaxSize
+		}
+		fieldValue := fl.Field().Int()
+		if fieldValue < 0 {
+			return false
+		}
+		return uint64(fieldValue) <= maxSize
+	}
+}
+
 func greatThanInt64PtrValidator() validator.Func {
 	return func(fl validator.FieldLevel) bool {
 		field := fl.Field()

diff --git a/cmd/compiler/main.go b/cmd/compiler/main.go
@@ -42,7 +42,7 @@ func main() {
 			config.WithRPC(cfg.RPC),
 			config.WithDatabase(cfg.DB),
 			config.WithRabbit(cfg.RabbitMQ, cfg.Compiler.ProjectName, cfg.Compiler.MQ),
-			config.WithStorage(cfg.Storage),
+			config.WithStorage(cfg.Storage, 0),
 			config.WithAWS(cfg.Compiler.AWS),
 		),
 	}

diff --git a/cmd/indexer/indexer/boost.go b/cmd/indexer/indexer/boost.go
@@ -132,7 +132,7 @@ func (bi *BoostIndexer) fetchExternalProtocols() error {
 // NewBoostIndexer -
 func NewBoostIndexer(cfg config.Config, network string, opts ...BoostIndexerOption) (*BoostIndexer, error) {
 	logger.WithNetwork(network).Info("Creating indexer object...")
-	es := core.WaitNew(cfg.Storage.URI, cfg.Storage.Timeout)
+	es := core.WaitNew(cfg.Storage.URI, cfg.Storage.Timeout, 0)
 
 	rpcProvider, ok := cfg.RPC[network]
 	if !ok {

diff --git a/cmd/metrics/main.go b/cmd/metrics/main.go
@@ -89,7 +89,7 @@ func main() {
 	}
 
 	configCtx := config.NewContext(
-		config.WithStorage(cfg.Storage),
+		config.WithStorage(cfg.Storage, 0),
 		config.WithRPC(cfg.RPC),
 		config.WithDatabase(cfg.DB),
 		config.WithRabbit(cfg.RabbitMQ, cfg.Metrics.ProjectName, cfg.Metrics.MQ),

diff --git a/configs/development.yml b/configs/development.yml
@@ -59,7 +59,7 @@ share_path: ${HOME}/.bcd
 base_url: http://localhost:8080
 
 ipfs:
-  - https://ipfs.baking-bad.org
+  - https://cloudflare-ipfs.com
   - https://ipfs.io
   - https://dweb.link
 
@@ -74,6 +74,7 @@ api:
   oauth_enabled: true
   sentry_enabled: false
   seed_enabled: false
+  page_size: ${PAGE_SIZE:-10}
   frontend:
     ga_enabled: false
     mempool_enabled: true

diff --git a/configs/production.yml b/configs/production.yml
@@ -60,7 +60,8 @@ share_path: /etc/bcd
 base_url: https://better-call.dev
 
 ipfs:
-  - https://ipfs.baking-bad.org
+  - ${SANDBOX_IPFS_GATEWAY:-https://ipfs.baking-bad.org}
+  - https://cloudflare-ipfs.com
   - https://ipfs.io
   - https://dweb.link
 
@@ -75,6 +76,7 @@ api:
   oauth_enabled: true
   sentry_enabled: true
   seed_enabled: false
+  page_size: ${PAGE_SIZE:-10}
   frontend:
     ga_enabled: true
     mempool_enabled: true

diff --git a/configs/sandbox.yml b/configs/sandbox.yml
@@ -31,6 +31,7 @@ api:
   oauth_enabled: false
   sentry_enabled: false
   seed_enabled: true
+  page_size: ${PAGE_SIZE:-10}
   frontend:
     ga_enabled: false
     mempool_enabled: false

diff --git a/configs/you.yml b/configs/you.yml
@@ -1,6 +1,6 @@
 rpc:
   mainnet:
-    uri: https://rpc.tzkt.io/mainnet
+    uri: https://mainnet-tezos.giganode.io
     timeout: 20
   edo2net:
     uri: https://rpc.tzkt.io/edo2net
@@ -9,18 +9,6 @@ rpc:
     uri: https://rpc.tzkt.io/florencenobanet
     timeout: 20
 
-tzkt:
-  mainnet:
-    uri: https://api.tzkt.io/v1/
-    services_uri: https://services.tzkt.io/v1/
-    base_uri: https://tzkt.io/
-    timeout: 20
-  edo2net:
-    uri: https://api.edo2net.tzkt.io/v1/
-    services_uri: https://services.tzkt.io/edo2net/v1/
-    base_uri: https://edo2net.tzkt.io/
-    timeout: 20
-
 storage:
   uri:
     - http://${ELASTIC_HOSTNAME:-elastic}:9200
@@ -59,7 +47,8 @@ share_path: /etc/bcd
 base_url: https://you.better-call.dev
 
 ipfs:
-  - https://ipfs.baking-bad.org
+  - ${SANDBOX_IPFS_GATEWAY:-https://ipfs.baking-bad.org}
+  - https://cloudflare-ipfs.com
   - https://ipfs.io
   - https://dweb.link
 
@@ -74,6 +63,7 @@ api:
   oauth_enabled: true
   sentry_enabled: true
   seed_enabled: false
+  page_size: ${PAGE_SIZE:-10}
   frontend:
     ga_enabled: false
     mempool_enabled: true
@@ -108,7 +98,6 @@ indexer:
     publisher: true
   networks:
     mainnet:
-      boost: tzkt
     edo2net:
     florencenet:
 

diff --git a/docker-compose.gateway.yml b/docker-compose.gateway.yml
@@ -45,13 +45,15 @@ services:
       context: .
       dockerfile: build/api/Dockerfile
     environment:
-      - BCD_ENV=production
+      - BCD_ENV=you
       - GIN_MODE=release
       - POSTGRES_USER=root
       - POSTGRES_PASSWORD=root
       - POSTGRES_DB=bcd
       - RABBITMQ_DEFAULT_USER=guest
       - RABBITMQ_DEFAULT_PASS=guest
+      - PAGE_SIZE=100
+      - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com
     depends_on:
       - elastic
       - db
@@ -69,12 +71,13 @@ services:
       context: .
       dockerfile: build/indexer/Dockerfile
     environment:
-      - BCD_ENV=production
+      - BCD_ENV=you
       - POSTGRES_USER=root
       - POSTGRES_PASSWORD=root
       - POSTGRES_DB=bcd
       - RABBITMQ_DEFAULT_USER=guest
       - RABBITMQ_DEFAULT_PASS=guest
+      - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com
     depends_on:
       - elastic
       - mq
@@ -90,12 +93,13 @@ services:
       context: .
       dockerfile: build/metrics/Dockerfile
     environment:
-      - BCD_ENV=production
+      - BCD_ENV=you
       - POSTGRES_USER=root
       - POSTGRES_PASSWORD=root
       - POSTGRES_DB=bcd
       - RABBITMQ_DEFAULT_USER=guest
       - RABBITMQ_DEFAULT_PASS=guest
+      - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com
     depends_on:
       - elastic
       - mq

diff --git a/docker-compose.sandbox.yml b/docker-compose.sandbox.yml
@@ -47,7 +47,7 @@ services:
       - RABBITMQ_DEFAULT_USER=guest
       - RABBITMQ_DEFAULT_PASS=guest
       - SANDBOX_NODE_URI=http://sandbox:8732
-      - SANDBOX_IPFS_GATEWAY=https://ipfs.io
+      - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com
     depends_on:
       - elastic
       - db
@@ -68,7 +68,7 @@ services:
       - RABBITMQ_DEFAULT_USER=guest
       - RABBITMQ_DEFAULT_PASS=guest
       - SANDBOX_NODE_URI=http://sandbox:8732
-      - SANDBOX_IPFS_GATEWAY=https://ipfs.io
+      - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com
     depends_on:
       - elastic
       - mq
@@ -90,7 +90,7 @@ services:
       - RABBITMQ_DEFAULT_USER=guest
       - RABBITMQ_DEFAULT_PASS=guest
       - SANDBOX_NODE_URI=http://sandbox:8732
-      - SANDBOX_IPFS_GATEWAY=https://ipfs.io
+      - SANDBOX_IPFS_GATEWAY=https://cloudflare-ipfs.com
     depends_on:
       - elastic
       - mq

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -32,20 +32,7 @@ type Config struct {
 	IPFSGateways []string              `yaml:"ipfs"`
 	Domains      TezosDomainsConfig    `yaml:"domains"`
 
-	API struct {
-		ProjectName   string         `yaml:"project_name"`
-		Bind          string         `yaml:"bind"`
-		SwaggerHost   string         `yaml:"swagger_host"`
-		CorsEnabled   bool           `yaml:"cors_enabled"`
-		OAuthEnabled  bool           `yaml:"oauth_enabled"`
-		SentryEnabled bool           `yaml:"sentry_enabled"`
-		SeedEnabled   bool           `yaml:"seed_enabled"`
-		Frontend      FrontendConfig `yaml:"frontend"`
-		Seed          SeedConfig     `yaml:"seed"`
-		Networks      []string       `yaml:"networks"`
-		MQ            MQConfig       `yaml:"mq"`
-		Pinata        PinataConfig   `yaml:"pinata"`
-	} `yaml:"api"`
+	API APIConfig `yaml:"api"`
 
 	Compiler struct {
 		ProjectName   string    `yaml:"project_name"`
@@ -171,6 +158,22 @@ type SeedConfig struct {
 	} `yaml:"accounts"`
 }
 
+type APIConfig struct {
+	ProjectName   string         `yaml:"project_name"`
+	Bind          string         `yaml:"bind"`
+	SwaggerHost   string         `yaml:"swagger_host"`
+	CorsEnabled   bool           `yaml:"cors_enabled"`
+	OAuthEnabled  bool           `yaml:"oauth_enabled"`
+	SentryEnabled bool           `yaml:"sentry_enabled"`
+	SeedEnabled   bool           `yaml:"seed_enabled"`
+	Frontend      FrontendConfig `yaml:"frontend"`
+	Seed          SeedConfig     `yaml:"seed"`
+	Networks      []string       `yaml:"networks"`
+	MQ            MQConfig       `yaml:"mq"`
+	Pinata        PinataConfig   `yaml:"pinata"`
+	PageSize      uint64         `yaml:"page_size"`
+}
+
 // SentryConfig -
 type SentryConfig struct {
 	Environment string `yaml:"environment"`

diff --git a/internal/config/options.go b/internal/config/options.go
@@ -64,7 +64,7 @@ func WithRPC(rpcConfig map[string]RPCConfig) ContextOption {
 }
 
 // WithStorage -
-func WithStorage(cfg StorageConfig) ContextOption {
+func WithStorage(cfg StorageConfig, maxPageSize int64) ContextOption {
 	return func(ctx *Context) {
 		if len(cfg.URI) == 0 {
 			panic("Please set connection strings to storage in config")
@@ -93,7 +93,7 @@ func WithStorage(cfg StorageConfig) ContextOption {
 				panic(err)
 			}
 		} else {
-			es := core.WaitNew(cfg.URI, cfg.Timeout)
+			es := core.WaitNew(cfg.URI, cfg.Timeout, maxPageSize)
 
 			ctx.Storage = es
 			ctx.BigMapActions = bigmapaction.NewStorage(es)