Feature: connections count in ORM pool (#768)

baking-bad · Sep 21, 2021 · 8546fbf · 8546fbf
1 parent 9c81394
commit 8546fbf
Show file tree

Hide file tree

Showing 13 changed files with 92 additions and 27 deletions.
diff --git a/cmd/api/handlers/context.go b/cmd/api/handlers/context.go
@@ -15,7 +15,7 @@ type Context struct {
 // NewContext -
 func NewContext(cfg config.Config) (*Context, error) {
 	ctx := config.NewContext(
-		config.WithStorage(cfg.Storage, cfg.API.ProjectName, int64(cfg.API.PageSize)),
+		config.WithStorage(cfg.Storage, cfg.API.ProjectName, int64(cfg.API.PageSize), cfg.API.Connections.Open, cfg.API.Connections.Idle),
 		config.WithRPC(cfg.RPC),
 		config.WithSearch(cfg.Storage),
 		config.WithShare(cfg.SharePath),

diff --git a/cmd/indexer/indexer/boost.go b/cmd/indexer/indexer/boost.go
@@ -115,7 +115,7 @@ func NewBoostIndexer(cfg config.Config, network types.Network, opts ...BoostInde
 
 	ctx := config.NewContext(
 		config.WithConfigCopy(cfg),
-		config.WithStorage(cfg.Storage, "indexer", 10),
+		config.WithStorage(cfg.Storage, "indexer", 10, cfg.Indexer.Connections.Open, cfg.Indexer.Connections.Idle),
 		config.WithSearch(cfg.Storage),
 		config.WithShare(cfg.SharePath),
 	)

diff --git a/cmd/metrics/main.go b/cmd/metrics/main.go
@@ -32,7 +32,7 @@ func main() {
 	}
 
 	ctx = config.NewContext(
-		config.WithStorage(cfg.Storage, cfg.Metrics.ProjectName, 0),
+		config.WithStorage(cfg.Storage, cfg.Metrics.ProjectName, 0, cfg.Metrics.Connections.Open, cfg.Metrics.Connections.Idle),
 		config.WithRPC(cfg.RPC),
 		config.WithSearch(cfg.Storage),
 		config.WithShare(cfg.SharePath),

diff --git a/configs/development.yml b/configs/development.yml
@@ -75,6 +75,9 @@ api:
     key: ${PINATA_KEY}
     secret_key: ${PINATA_SECRET_KEY}
     timeout_seconds: 10
+  connections:
+    max: 50
+    idle: 50
 
 indexer:
   project_name: indexer
@@ -87,11 +90,17 @@ indexer:
       boost: tzkt
     florencenet:
     granadanet:
+  connections:
+    max: 5
+    idle: 5
 
 metrics:
   project_name: metrics
   sentry_enabled: false
   cache_aliases_seconds: 30
+  connections:
+    max: 10
+    idle: 10
 
 scripts:
   aws:
@@ -104,6 +113,9 @@ scripts:
     - edo2net
     - florencenet
     - granadanet
+  connections:
+    max: 5
+    idle: 5
 
 graphql:
   db: "host=127.0.0.1 port=5432 user=graphql dbname=indexer password=${POSTGRES_GRAPHQL} sslmode=disable"
diff --git a/configs/production.yml b/configs/production.yml
@@ -62,6 +62,10 @@ api:
     key: ${PINATA_KEY}
     secret_key: ${PINATA_SECRET_KEY}
     timeout_seconds: 10
+  connections:
+    max: 50
+    idle: 10
+
 
 indexer:
   project_name: indexer
@@ -72,11 +76,17 @@ indexer:
       boost: tzkt
     florencenet:
     granadanet:
+  connections:
+    max: 5
+    idle: 5
 
 metrics:
   project_name: metrics
   sentry_enabled: true
   cache_aliases_seconds: 30
+  connections:
+    max: 20
+    idle: 20
 
 scripts:
   aws:
@@ -89,6 +99,9 @@ scripts:
     - edo2net
     - florencenet
     - granadanet
+  connections:
+    max: 5
+    idle: 5
 
 graphql:
   db: "host=${DB_HOSTNAME} port=5432 user=graphql dbname=indexer password=${POSTGRES_GRAPHQL} sslmode=disable"
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -35,21 +35,23 @@ type Config struct {
 		Networks map[string]struct {
 			Boost string `yaml:"boost"`
 		} `yaml:"networks"`
-		ProjectName   string `yaml:"project_name"`
-		SentryEnabled bool   `yaml:"sentry_enabled"`
-
-		SkipDelegatorBlocks bool `yaml:"skip_delegator_blocks"`
+		ProjectName         string      `yaml:"project_name"`
+		SentryEnabled       bool        `yaml:"sentry_enabled"`
+		SkipDelegatorBlocks bool        `yaml:"skip_delegator_blocks"`
+		Connections         Connections `yaml:"connections"`
 	} `yaml:"indexer"`
 
 	Metrics struct {
-		ProjectName         string `yaml:"project_name"`
-		SentryEnabled       bool   `yaml:"sentry_enabled"`
-		CacheAliasesSeconds int    `yaml:"cache_aliases_seconds"`
+		ProjectName         string      `yaml:"project_name"`
+		SentryEnabled       bool        `yaml:"sentry_enabled"`
+		CacheAliasesSeconds int         `yaml:"cache_aliases_seconds"`
+		Connections         Connections `yaml:"connections"`
 	} `yaml:"metrics"`
 
 	Scripts struct {
-		AWS      AWSConfig `yaml:"aws"`
-		Networks []string  `yaml:"networks"`
+		AWS         AWSConfig   `yaml:"aws"`
+		Networks    []string    `yaml:"networks"`
+		Connections Connections `yaml:"connections"`
 	} `yaml:"scripts"`
 
 	GraphQL struct {
@@ -156,6 +158,7 @@ type APIConfig struct {
 	Networks      []string       `yaml:"networks"`
 	Pinata        PinataConfig   `yaml:"pinata"`
 	PageSize      uint64         `yaml:"page_size"`
+	Connections   Connections    `yaml:"connections"`
 }
 
 // SentryConfig -
@@ -176,6 +179,12 @@ type PinataConfig struct {
 	TimeoutSeconds int    `yaml:"timeout_seconds"`
 }
 
+// Connections -
+type Connections struct {
+	Open int `yaml:"open"`
+	Idle int `yaml:"idle"`
+}
+
 // LoadDefaultConfig -
 func LoadDefaultConfig() (Config, error) {
 	configurations := map[string]string{

diff --git a/internal/config/options.go b/internal/config/options.go
@@ -54,13 +54,17 @@ func WithRPC(rpcConfig map[string]RPCConfig) ContextOption {
 }
 
 // WithStorage -
-func WithStorage(cfg StorageConfig, appName string, maxPageSize int64) ContextOption {
+func WithStorage(cfg StorageConfig, appName string, maxPageSize int64, maxConnCount, idleConnCount int) ContextOption {
 	return func(ctx *Context) {
 		if len(cfg.Elastic) == 0 {
 			panic("Please set connection strings to storage in config")
 		}
 
-		pg := pgCore.WaitNew(cfg.Postgres, appName, cfg.Timeout, pgCore.WithPageSize(maxPageSize))
+		pg := pgCore.WaitNew(cfg.Postgres, appName, cfg.Timeout,
+			pgCore.WithPageSize(maxPageSize),
+			pgCore.WithIdleConnections(idleConnCount),
+			pgCore.WithMaxConnections(maxConnCount),
+		)
 		ctx.StorageDB = pg
 		ctx.Storage = pg
 		ctx.BigMapActions = bigmapaction.NewStorage(pg)

diff --git a/internal/postgres/core/options.go b/internal/postgres/core/options.go
@@ -1,6 +1,9 @@
 package core
 
 import (
+	"time"
+
+	bcdLogger "github.com/baking-bad/bcdhub/internal/logger"
 	"github.com/baking-bad/bcdhub/internal/postgres/consts"
 )
 
@@ -16,3 +19,35 @@ func WithPageSize(pageSize int64) PostgresOption {
 		pg.PageSize = pageSize
 	}
 }
+
+// WithMaxConnections -
+func WithMaxConnections(count int) PostgresOption {
+	return func(pg *Postgres) {
+		if count == 0 {
+			count = consts.DefaultSize
+		}
+		sql, err := pg.DB.DB()
+		if err != nil {
+			bcdLogger.Err(err)
+			return
+		}
+		sql.SetMaxOpenConns(count)
+		sql.SetConnMaxLifetime(time.Hour)
+	}
+}
+
+// WithIdleConnections -
+func WithIdleConnections(count int) PostgresOption {
+	return func(pg *Postgres) {
+		if count == 0 {
+			count = consts.DefaultSize
+		}
+		sql, err := pg.DB.DB()
+		if err != nil {
+			bcdLogger.Err(err)
+			return
+		}
+		sql.SetMaxIdleConns(count)
+		sql.SetConnMaxIdleTime(time.Minute * 30)
+	}
+}
diff --git a/internal/postgres/core/postgres.go b/internal/postgres/core/postgres.go
@@ -38,14 +38,6 @@ func New(connection, appName string, opts ...PostgresOption) (*Postgres, error)
 		opt(&pg)
 	}
 
-	sql, err := pg.DB.DB()
-	if err != nil {
-		return nil, err
-	}
-
-	sql.SetMaxOpenConns(50)
-	sql.SetMaxIdleConns(25)
-
 	return &pg, nil
 }
 

diff --git a/scripts/api_tester/main.go b/scripts/api_tester/main.go
@@ -22,7 +22,7 @@ func main() {
 	}
 
 	ctx := config.NewContext(
-		config.WithStorage(cfg.Storage, "api_tester", 0),
+		config.WithStorage(cfg.Storage, "api_tester", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle),
 		config.WithRPC(cfg.RPC),
 		config.WithShare(cfg.SharePath),
 		config.WithTzKTServices(cfg.TzKT),
@@ -55,7 +55,7 @@ func request(uri string) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("Invalid status code: [%d] %s", resp.StatusCode, url)
+		return errors.Errorf("Invalid status code: [%d] %s", resp.StatusCode, url)
 	}
 	return nil
 }
diff --git a/scripts/bcdctl/main.go b/scripts/bcdctl/main.go
@@ -31,7 +31,7 @@ func main() {
 	}
 
 	ctx = config.NewContext(
-		config.WithStorage(cfg.Storage, "bcdctl", 0),
+		config.WithStorage(cfg.Storage, "bcdctl", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle),
 		config.WithConfigCopy(cfg),
 		config.WithRPC(cfg.RPC),
 		config.WithShare(cfg.SharePath),

diff --git a/scripts/migration/main.go b/scripts/migration/main.go
@@ -55,7 +55,7 @@ func main() {
 
 	ctx := config.NewContext(
 		config.WithShare(cfg.SharePath),
-		config.WithStorage(cfg.Storage, "migrations", 0),
+		config.WithStorage(cfg.Storage, "migrations", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle),
 		config.WithRPC(cfg.RPC),
 		config.WithConfigCopy(cfg),
 		config.WithLoadErrorDescriptions(),

diff --git a/scripts/nginx/main.go b/scripts/nginx/main.go
@@ -17,7 +17,7 @@ func main() {
 	}
 
 	ctx := config.NewContext(
-		config.WithStorage(cfg.Storage, "nginx", 0),
+		config.WithStorage(cfg.Storage, "nginx", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle),
 		config.WithConfigCopy(cfg),
 	)
 	defer ctx.Close()