diff --git a/cmd/api/handlers/context.go b/cmd/api/handlers/context.go index 5b5d0a272..28bc4dc76 100644 --- a/cmd/api/handlers/context.go +++ b/cmd/api/handlers/context.go @@ -15,7 +15,7 @@ type Context struct { // NewContext - func NewContext(cfg config.Config) (*Context, error) { ctx := config.NewContext( - config.WithStorage(cfg.Storage, cfg.API.ProjectName, int64(cfg.API.PageSize)), + config.WithStorage(cfg.Storage, cfg.API.ProjectName, int64(cfg.API.PageSize), cfg.API.Connections.Open, cfg.API.Connections.Idle), config.WithRPC(cfg.RPC), config.WithSearch(cfg.Storage), config.WithShare(cfg.SharePath), diff --git a/cmd/indexer/indexer/boost.go b/cmd/indexer/indexer/boost.go index ae0acef45..57b95b835 100644 --- a/cmd/indexer/indexer/boost.go +++ b/cmd/indexer/indexer/boost.go @@ -115,7 +115,7 @@ func NewBoostIndexer(cfg config.Config, network types.Network, opts ...BoostInde ctx := config.NewContext( config.WithConfigCopy(cfg), - config.WithStorage(cfg.Storage, "indexer", 10), + config.WithStorage(cfg.Storage, "indexer", 10, cfg.Indexer.Connections.Open, cfg.Indexer.Connections.Idle), config.WithSearch(cfg.Storage), config.WithShare(cfg.SharePath), ) diff --git a/cmd/metrics/main.go b/cmd/metrics/main.go index 70e9fd538..060b45624 100644 --- a/cmd/metrics/main.go +++ b/cmd/metrics/main.go @@ -32,7 +32,7 @@ func main() { } ctx = config.NewContext( - config.WithStorage(cfg.Storage, cfg.Metrics.ProjectName, 0), + config.WithStorage(cfg.Storage, cfg.Metrics.ProjectName, 0, cfg.Metrics.Connections.Open, cfg.Metrics.Connections.Idle), config.WithRPC(cfg.RPC), config.WithSearch(cfg.Storage), config.WithShare(cfg.SharePath), diff --git a/configs/development.yml b/configs/development.yml index 067a7cfef..d3e1fe4c6 100644 --- a/configs/development.yml +++ b/configs/development.yml @@ -75,6 +75,9 @@ api: key: ${PINATA_KEY} secret_key: ${PINATA_SECRET_KEY} timeout_seconds: 10 + connections: + max: 50 + idle: 50 indexer: project_name: indexer @@ -87,11 +90,17 @@ indexer: boost: tzkt florencenet: granadanet: + connections: + max: 5 + idle: 5 metrics: project_name: metrics sentry_enabled: false cache_aliases_seconds: 30 + connections: + max: 10 + idle: 10 scripts: aws: @@ -104,6 +113,9 @@ scripts: - edo2net - florencenet - granadanet + connections: + max: 5 + idle: 5 graphql: db: "host=127.0.0.1 port=5432 user=graphql dbname=indexer password=${POSTGRES_GRAPHQL} sslmode=disable" \ No newline at end of file diff --git a/configs/production.yml b/configs/production.yml index 63f8792c1..96b33d9a3 100644 --- a/configs/production.yml +++ b/configs/production.yml @@ -62,6 +62,10 @@ api: key: ${PINATA_KEY} secret_key: ${PINATA_SECRET_KEY} timeout_seconds: 10 + connections: + max: 50 + idle: 10 + indexer: project_name: indexer @@ -72,11 +76,17 @@ indexer: boost: tzkt florencenet: granadanet: + connections: + max: 5 + idle: 5 metrics: project_name: metrics sentry_enabled: true cache_aliases_seconds: 30 + connections: + max: 20 + idle: 20 scripts: aws: @@ -89,6 +99,9 @@ scripts: - edo2net - florencenet - granadanet + connections: + max: 5 + idle: 5 graphql: db: "host=${DB_HOSTNAME} port=5432 user=graphql dbname=indexer password=${POSTGRES_GRAPHQL} sslmode=disable" diff --git a/internal/config/config.go b/internal/config/config.go index 5941c5a42..d6bf65d2c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,21 +35,23 @@ type Config struct { Networks map[string]struct { Boost string `yaml:"boost"` } `yaml:"networks"` - ProjectName string `yaml:"project_name"` - SentryEnabled bool `yaml:"sentry_enabled"` - - SkipDelegatorBlocks bool `yaml:"skip_delegator_blocks"` + ProjectName string `yaml:"project_name"` + SentryEnabled bool `yaml:"sentry_enabled"` + SkipDelegatorBlocks bool `yaml:"skip_delegator_blocks"` + Connections Connections `yaml:"connections"` } `yaml:"indexer"` Metrics struct { - ProjectName string `yaml:"project_name"` - SentryEnabled bool `yaml:"sentry_enabled"` - CacheAliasesSeconds int `yaml:"cache_aliases_seconds"` + ProjectName string `yaml:"project_name"` + SentryEnabled bool `yaml:"sentry_enabled"` + CacheAliasesSeconds int `yaml:"cache_aliases_seconds"` + Connections Connections `yaml:"connections"` } `yaml:"metrics"` Scripts struct { - AWS AWSConfig `yaml:"aws"` - Networks []string `yaml:"networks"` + AWS AWSConfig `yaml:"aws"` + Networks []string `yaml:"networks"` + Connections Connections `yaml:"connections"` } `yaml:"scripts"` GraphQL struct { @@ -156,6 +158,7 @@ type APIConfig struct { Networks []string `yaml:"networks"` Pinata PinataConfig `yaml:"pinata"` PageSize uint64 `yaml:"page_size"` + Connections Connections `yaml:"connections"` } // SentryConfig - @@ -176,6 +179,12 @@ type PinataConfig struct { TimeoutSeconds int `yaml:"timeout_seconds"` } +// Connections - +type Connections struct { + Open int `yaml:"open"` + Idle int `yaml:"idle"` +} + // LoadDefaultConfig - func LoadDefaultConfig() (Config, error) { configurations := map[string]string{ diff --git a/internal/config/options.go b/internal/config/options.go index b9074b1de..396eb9414 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -54,13 +54,17 @@ func WithRPC(rpcConfig map[string]RPCConfig) ContextOption { } // WithStorage - -func WithStorage(cfg StorageConfig, appName string, maxPageSize int64) ContextOption { +func WithStorage(cfg StorageConfig, appName string, maxPageSize int64, maxConnCount, idleConnCount int) ContextOption { return func(ctx *Context) { if len(cfg.Elastic) == 0 { panic("Please set connection strings to storage in config") } - pg := pgCore.WaitNew(cfg.Postgres, appName, cfg.Timeout, pgCore.WithPageSize(maxPageSize)) + pg := pgCore.WaitNew(cfg.Postgres, appName, cfg.Timeout, + pgCore.WithPageSize(maxPageSize), + pgCore.WithIdleConnections(idleConnCount), + pgCore.WithMaxConnections(maxConnCount), + ) ctx.StorageDB = pg ctx.Storage = pg ctx.BigMapActions = bigmapaction.NewStorage(pg) diff --git a/internal/postgres/core/options.go b/internal/postgres/core/options.go index 2f2633c55..8990b49ae 100644 --- a/internal/postgres/core/options.go +++ b/internal/postgres/core/options.go @@ -1,6 +1,9 @@ package core import ( + "time" + + bcdLogger "github.com/baking-bad/bcdhub/internal/logger" "github.com/baking-bad/bcdhub/internal/postgres/consts" ) @@ -16,3 +19,35 @@ func WithPageSize(pageSize int64) PostgresOption { pg.PageSize = pageSize } } + +// WithMaxConnections - +func WithMaxConnections(count int) PostgresOption { + return func(pg *Postgres) { + if count == 0 { + count = consts.DefaultSize + } + sql, err := pg.DB.DB() + if err != nil { + bcdLogger.Err(err) + return + } + sql.SetMaxOpenConns(count) + sql.SetConnMaxLifetime(time.Hour) + } +} + +// WithIdleConnections - +func WithIdleConnections(count int) PostgresOption { + return func(pg *Postgres) { + if count == 0 { + count = consts.DefaultSize + } + sql, err := pg.DB.DB() + if err != nil { + bcdLogger.Err(err) + return + } + sql.SetMaxIdleConns(count) + sql.SetConnMaxIdleTime(time.Minute * 30) + } +} diff --git a/internal/postgres/core/postgres.go b/internal/postgres/core/postgres.go index a27e42873..ca4a146ec 100644 --- a/internal/postgres/core/postgres.go +++ b/internal/postgres/core/postgres.go @@ -38,14 +38,6 @@ func New(connection, appName string, opts ...PostgresOption) (*Postgres, error) opt(&pg) } - sql, err := pg.DB.DB() - if err != nil { - return nil, err - } - - sql.SetMaxOpenConns(50) - sql.SetMaxIdleConns(25) - return &pg, nil } diff --git a/scripts/api_tester/main.go b/scripts/api_tester/main.go index b46788282..e7acaf91f 100644 --- a/scripts/api_tester/main.go +++ b/scripts/api_tester/main.go @@ -22,7 +22,7 @@ func main() { } ctx := config.NewContext( - config.WithStorage(cfg.Storage, "api_tester", 0), + config.WithStorage(cfg.Storage, "api_tester", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle), config.WithRPC(cfg.RPC), config.WithShare(cfg.SharePath), config.WithTzKTServices(cfg.TzKT), @@ -55,7 +55,7 @@ func request(uri string) error { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Invalid status code: [%d] %s", resp.StatusCode, url) + return errors.Errorf("Invalid status code: [%d] %s", resp.StatusCode, url) } return nil } diff --git a/scripts/bcdctl/main.go b/scripts/bcdctl/main.go index 93aab2b07..841ec7754 100644 --- a/scripts/bcdctl/main.go +++ b/scripts/bcdctl/main.go @@ -31,7 +31,7 @@ func main() { } ctx = config.NewContext( - config.WithStorage(cfg.Storage, "bcdctl", 0), + config.WithStorage(cfg.Storage, "bcdctl", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle), config.WithConfigCopy(cfg), config.WithRPC(cfg.RPC), config.WithShare(cfg.SharePath), diff --git a/scripts/migration/main.go b/scripts/migration/main.go index 3579a5e31..d410a2910 100644 --- a/scripts/migration/main.go +++ b/scripts/migration/main.go @@ -55,7 +55,7 @@ func main() { ctx := config.NewContext( config.WithShare(cfg.SharePath), - config.WithStorage(cfg.Storage, "migrations", 0), + config.WithStorage(cfg.Storage, "migrations", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle), config.WithRPC(cfg.RPC), config.WithConfigCopy(cfg), config.WithLoadErrorDescriptions(), diff --git a/scripts/nginx/main.go b/scripts/nginx/main.go index 7a405a174..8acb27dd0 100644 --- a/scripts/nginx/main.go +++ b/scripts/nginx/main.go @@ -17,7 +17,7 @@ func main() { } ctx := config.NewContext( - config.WithStorage(cfg.Storage, "nginx", 0), + config.WithStorage(cfg.Storage, "nginx", 0, cfg.Scripts.Connections.Open, cfg.Scripts.Connections.Idle), config.WithConfigCopy(cfg), ) defer ctx.Close()