Skip to content

Commit

Permalink
feat(api): add configurable page sizes (#628)
Browse files Browse the repository at this point in the history
  • Loading branch information
@gfyrag
gfyrag authored Jan 2, 2025
1 parent 8cf1718 commit b1cb3cf
Show file tree
Hide file tree
Showing 19 changed files with 247 additions and 177 deletions.
33 changes: 26 additions & 7 deletions cmd/serve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"github.com/formancehq/go-libs/v2/logging"
"github.com/formancehq/ledger/internal/api/common"
systemstore "github.com/formancehq/ledger/internal/storage/system"
"net/http"
"net/http/pprof"
Expand Down Expand Up @@ -43,6 +44,8 @@ const (
BulkMaxSizeFlag = "bulk-max-size"
BulkParallelFlag = "bulk-parallel"
NumscriptInterpreterFlag = "experimental-numscript-interpreter"
DefaultPageSizeFlag = "default-page-size"
MaxPageSizeFlag = "max-page-size"
)

func NewServeCommand() *cobra.Command {
Expand Down Expand Up @@ -73,6 +76,16 @@ func NewServeCommand() *cobra.Command {
return err
}

maxPageSize, err := cmd.Flags().GetUint64(MaxPageSizeFlag)
if err != nil {
return err
}

defaultPageSize, err := cmd.Flags().GetUint64(DefaultPageSizeFlag)
if err != nil {
return err
}

options := []fx.Option{
fx.NopLogger,
otlp.FXModuleFromFlags(cmd),
Expand Down Expand Up @@ -102,18 +115,22 @@ func NewServeCommand() *cobra.Command {
MaxSize: bulkMaxSize,
Parallel: bulkParallel,
},
Pagination: common.PaginationConfig{
MaxPageSize: maxPageSize,
DefaultPageSize: defaultPageSize,
},
}),
fx.Decorate(func(
params struct {
fx.In
fx.In

Handler chi.Router
HealthController *health.HealthController
Logger logging.Logger
Handler chi.Router
HealthController *health.HealthController
Logger logging.Logger

MeterProvider *metric.MeterProvider `optional:"true"`
Exporter *otlpmetrics.InMemoryExporter `optional:"true"`
},
MeterProvider *metric.MeterProvider `optional:"true"`
Exporter *otlpmetrics.InMemoryExporter `optional:"true"`
},
) chi.Router {
return assembleFinalRouter(
service.IsDebug(cmd),
Expand All @@ -140,6 +157,8 @@ func NewServeCommand() *cobra.Command {
cmd.Flags().Int(BulkMaxSizeFlag, api.DefaultBulkMaxSize, "Bulk max size (default 100)")
cmd.Flags().Int(BulkParallelFlag, 10, "Bulk max parallelism")
cmd.Flags().Bool(NumscriptInterpreterFlag, false, "Enable experimental numscript rewrite")
cmd.Flags().Uint64(MaxPageSizeFlag, 100, "Max page size")
cmd.Flags().Uint64(DefaultPageSizeFlag, 15, "Default page size")

service.AddFlags(cmd.Flags())
bunconnect.AddFlags(cmd.Flags())
Expand Down
6 changes: 6 additions & 0 deletions internal/api/common/pagination.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package common

type PaginationConfig struct {
MaxPageSize uint64
DefaultPageSize uint64
}
9 changes: 6 additions & 3 deletions internal/api/module.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/formancehq/go-libs/v2/auth"
"github.com/formancehq/go-libs/v2/health"
"github.com/formancehq/ledger/internal/api/bulking"
"github.com/formancehq/ledger/internal/api/common"
"github.com/formancehq/ledger/internal/controller/system"
"github.com/go-chi/chi/v5"
"go.opentelemetry.io/otel/trace"
Expand All @@ -17,9 +18,10 @@ type BulkConfig struct {
}

type Config struct {
Version string
Debug bool
Bulk BulkConfig
Version string
Debug bool
Bulk BulkConfig
Pagination common.PaginationConfig
}

func Module(cfg Config) fx.Option {
Expand All @@ -40,6 +42,7 @@ func Module(cfg Config) fx.Option {
bulking.WithParallelism(cfg.Bulk.Parallel),
bulking.WithTracer(tracerProvider.Tracer("api.bulking")),
)),
WithPaginationConfiguration(cfg.Pagination),
)
}),
health.Module(),
Expand Down
15 changes: 14 additions & 1 deletion internal/api/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package api

import (
"github.com/formancehq/go-libs/v2/api"
"github.com/formancehq/go-libs/v2/bun/bunpaginate"
"github.com/formancehq/ledger/internal/api/bulking"
"github.com/formancehq/ledger/internal/controller/system"
"go.opentelemetry.io/otel/attribute"
Expand Down Expand Up @@ -71,6 +72,7 @@ func NewRouter(
"application/json": bulking.NewJSONBulkHandlerFactory(routerOptions.bulkMaxSize),
"application/vnd.formance.ledger.api.v2.bulk+script-stream": bulking.NewScriptStreamBulkHandlerFactory(),
}),
v2.WithPaginationConfig(routerOptions.paginationConfig),
)
mux.Handle("/v2*", http.StripPrefix("/v2", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
chi.RouteContext(r.Context()).Reset()
Expand All @@ -91,7 +93,8 @@ func NewRouter(
type routerOptions struct {
tracer trace.Tracer
bulkMaxSize int
bulkerFactory bulking.BulkerFactory
bulkerFactory bulking.BulkerFactory
paginationConfig common.PaginationConfig
}

type RouterOption func(ro *routerOptions)
Expand All @@ -114,9 +117,19 @@ func WithBulkerFactory(bf bulking.BulkerFactory) RouterOption {
}
}

func WithPaginationConfiguration(paginationConfig common.PaginationConfig) RouterOption {
return func(ro *routerOptions) {
ro.paginationConfig = paginationConfig
}
}

var defaultRouterOptions = []RouterOption{
WithTracer(nooptracer.Tracer{}),
WithBulkMaxSize(DefaultBulkMaxSize),
WithPaginationConfiguration(common.PaginationConfig{
MaxPageSize: bunpaginate.MaxPageSize,
DefaultPageSize: bunpaginate.QueryDefaultPageSize,
}),
}

const DefaultBulkMaxSize = 100
17 changes: 13 additions & 4 deletions internal/api/v2/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package v2

import (
. "github.com/formancehq/go-libs/v2/collectionutils"
"github.com/formancehq/ledger/internal/api/common"
ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger"
"io"
"net/http"
Expand Down Expand Up @@ -61,14 +62,18 @@ func getExpand(r *http.Request) []string {
)
}

func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) {
func getOffsetPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) {
return bunpaginate.Extract[ledgercontroller.OffsetPaginatedQuery[v]](r, func() (*ledgercontroller.OffsetPaginatedQuery[v], error) {
rq, err := getResourceQuery[v](r, modifiers...)
if err != nil {
return nil, err
}

pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize))
pageSize, err := bunpaginate.GetPageSize(
r,
bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize),
bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize),
)
if err != nil {
return nil, err
}
Expand All @@ -80,14 +85,18 @@ func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error
})
}

func getColumnPaginatedQuery[v any](r *http.Request, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) {
func getColumnPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) {
return bunpaginate.Extract[ledgercontroller.ColumnPaginatedQuery[v]](r, func() (*ledgercontroller.ColumnPaginatedQuery[v], error) {
rq, err := getResourceQuery[v](r, modifiers...)
if err != nil {
return nil, err
}

pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize))
pageSize, err := bunpaginate.GetPageSize(
r,
bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize),
bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize),
)
if err != nil {
return nil, err
}
Expand Down
39 changes: 20 additions & 19 deletions internal/api/v2/controllers_accounts_list.go
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,34 @@
package v2

import (
"net/http"

"errors"
"github.com/formancehq/go-libs/v2/api"
"github.com/formancehq/ledger/internal/api/common"
ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger"
"net/http"
)

func listAccounts(w http.ResponseWriter, r *http.Request) {
l := common.LedgerFromContext(r.Context())

query, err := getOffsetPaginatedQuery[any](r)
if err != nil {
api.BadRequest(w, common.ErrValidation, err)
return
}
func listAccounts(paginationConfig common.PaginationConfig) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
l := common.LedgerFromContext(r.Context())

cursor, err := l.ListAccounts(r.Context(), *query)
if err != nil {
switch {
case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}):
query, err := getOffsetPaginatedQuery[any](r, paginationConfig)
if err != nil {
api.BadRequest(w, common.ErrValidation, err)
default:
common.HandleCommonErrors(w, r, err)
return
}

cursor, err := l.ListAccounts(r.Context(), *query)
if err != nil {
switch {
case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}):
api.BadRequest(w, common.ErrValidation, err)
default:
common.HandleCommonErrors(w, r, err)
}
return
}
return
}

api.RenderCursor(w, *cursor)
api.RenderCursor(w, *cursor)
}
}
22 changes: 11 additions & 11 deletions internal/api/v2/controllers_accounts_list_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func TestAccountsList(t *testing.T) {
{
name: "nominal",
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Expand: make([]string, 0),
Expand All @@ -54,7 +54,7 @@ func TestAccountsList(t *testing.T) {
body: `{"$match": { "metadata[roles]": "admin" }}`,
expectBackendCall: true,
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Builder: query.Match("metadata[roles]", "admin"),
Expand All @@ -67,7 +67,7 @@ func TestAccountsList(t *testing.T) {
body: `{"$match": { "address": "foo" }}`,
expectBackendCall: true,
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Builder: query.Match("address", "foo"),
Expand All @@ -80,12 +80,12 @@ func TestAccountsList(t *testing.T) {
expectBackendCall: true,
queryParams: url.Values{
"cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{},
})},
},
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{},
},
},
Expand All @@ -112,7 +112,7 @@ func TestAccountsList(t *testing.T) {
"pageSize": []string{"1000000"},
},
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: MaxPageSize,
PageSize: bunpaginate.MaxPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Expand: make([]string, 0),
Expand All @@ -124,7 +124,7 @@ func TestAccountsList(t *testing.T) {
expectBackendCall: true,
body: `{"$lt": { "balance[USD/2]": 100 }}`,
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Builder: query.Lt("balance[USD/2]", float64(100)),
Expand All @@ -137,7 +137,7 @@ func TestAccountsList(t *testing.T) {
expectBackendCall: true,
body: `{"$exists": { "metadata": "foo" }}`,
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Builder: query.Exists("metadata", "foo"),
Expand All @@ -158,7 +158,7 @@ func TestAccountsList(t *testing.T) {
expectBackendCall: true,
returnErr: ledgercontroller.ErrInvalidQuery{},
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Expand: make([]string, 0),
Expand All @@ -172,7 +172,7 @@ func TestAccountsList(t *testing.T) {
expectBackendCall: true,
returnErr: ledgercontroller.ErrMissingFeature{},
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Expand: make([]string, 0),
Expand All @@ -186,7 +186,7 @@ func TestAccountsList(t *testing.T) {
expectBackendCall: true,
returnErr: errors.New("undefined error"),
expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
PageSize: DefaultPageSize,
PageSize: bunpaginate.QueryDefaultPageSize,
Options: ledgercontroller.ResourceQuery[any]{
PIT: &before,
Expand: make([]string, 0),
Expand Down
7 changes: 5 additions & 2 deletions internal/api/v2/controllers_ledgers_list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,14 @@ import (
"github.com/formancehq/ledger/internal/controller/system"
)

func listLedgers(b system.Controller) http.HandlerFunc {
func listLedgers(b system.Controller, paginationConfig common.PaginationConfig) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {

query, err := bunpaginate.Extract[ledgercontroller.ListLedgersQuery](r, func() (*ledgercontroller.ListLedgersQuery, error) {
pageSize, err := bunpaginate.GetPageSize(r)
pageSize, err := bunpaginate.GetPageSize(r,
bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize),
bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize),
)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions internal/api/v2/controllers_ledgers_list_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,15 @@ func TestListLedgers(t *testing.T) {
expectedErrorCode: common.ErrValidation,
expectBackendCall: true,
returnErr: ledgercontroller.ErrInvalidQuery{},
expectQuery: ledgercontroller.NewListLedgersQuery(DefaultPageSize),
expectQuery: ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize),
},
{
name: "with missing feature",
expectedStatusCode: http.StatusBadRequest,
expectedErrorCode: common.ErrValidation,
expectBackendCall: true,
returnErr: ledgercontroller.ErrMissingFeature{},
expectQuery: ledgercontroller.NewListLedgersQuery(DefaultPageSize),
expectQuery: ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize),
},
} {
t.Run(tc.name, func(t *testing.T) {
Expand Down
Loading

0 comments on commit b1cb3cf

Please sign in to comment.