From 32d22503503fbc3df06788c2c602608d1465599e Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 23 Oct 2024 18:49:11 +0200 Subject: [PATCH] feat: prevent downgrades of application --- docs/api/README.md | 2 +- go.mod | 14 +-- go.sum | 37 ++++--- internal/storage/driver/driver.go | 42 +++++++- internal/storage/driver/errors.go | 24 +++++ internal/storage/driver/migrations.go | 23 +++++ openapi.yaml | 4 +- openapi/v2.yaml | 4 +- pkg/client/.speakeasy/gen.lock | 6 +- pkg/client/.speakeasy/gen.yaml | 2 +- .../docs/models/components/v2migrationinfo.md | 2 +- pkg/client/formance.go | 4 +- .../models/components/v2migrationinfo.go | 4 +- pkg/testserver/api.go | 4 + pkg/testserver/helpers.go | 23 +++++ pkg/testserver/server.go | 97 +++++++++++-------- test/e2e/api_accounts_metadata_test.go | 2 +- test/e2e/api_ledgers_import_test.go | 2 +- test/e2e/api_transactions_create_test.go | 2 +- test/e2e/api_transactions_revert_test.go | 2 +- test/e2e/lifecycle_test.go | 58 ++++++++++- test/performance/env_testserver_test.go | 3 +- 22 files changed, 265 insertions(+), 96 deletions(-) create mode 100644 internal/storage/driver/errors.go diff --git a/docs/api/README.md b/docs/api/README.md index 063650048..eb3ee14a7 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -3277,7 +3277,7 @@ Authorization ( Scopes: ledger:write ) |Name|Type|Required|Restrictions|Description| |---|---|---|---|---| -|version|integer(int64)|false|none|none| +|version|string|false|none|none| |name|string|false|none|none| |date|string(date-time)|false|none|none| |state|string|false|none|none| diff --git a/go.mod b/go.mod index dcbc29388..f6747cd0e 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/formancehq/ledger go 1.22.1 -toolchain go1.22.7 +toolchain go1.23.2 replace github.com/formancehq/stack/ledger/client => ./pkg/client @@ -13,15 +13,13 @@ require ( github.com/alitto/pond v1.9.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 github.com/bluele/gcache v0.0.2 - github.com/formancehq/go-libs/v2 v2.0.1-0.20241022185745-110c95803b63 - github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 + github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163904-e440de7907c7 github.com/formancehq/stack/ledger/client v0.0.0-00010101000000-000000000000 github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/cors v1.2.1 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/invopop/jsonschema v0.12.0 - github.com/jackc/pgx/v5 v5.7.1 github.com/jamiealquiza/tachymeter v2.0.0+incompatible github.com/logrusorgru/aurora v2.0.3+incompatible github.com/nats-io/nats.go v1.37.0 @@ -29,7 +27,6 @@ require ( github.com/onsi/gomega v1.34.2 github.com/ory/dockertest/v3 v3.11.0 github.com/pborman/uuid v1.2.1 - github.com/pkg/errors v0.9.1 github.com/shomali11/xsql v0.0.0-20190608141458-bf76292144df github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 @@ -42,7 +39,7 @@ require ( go.opentelemetry.io/otel/metric v1.31.0 go.opentelemetry.io/otel/sdk/metric v1.31.0 go.opentelemetry.io/otel/trace v1.31.0 - go.uber.org/fx v1.23.0 + go.uber.org/fx v1.22.2 go.uber.org/mock v0.4.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 @@ -91,6 +88,8 @@ require ( github.com/ebitengine/purego v0.8.0 // indirect github.com/ericlagergren/decimal v0.0.0-20240411145413-00de7ca16731 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 + github.com/getsentry/sentry-go v0.28.1 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-chi/render v1.0.3 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -115,6 +114,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect @@ -140,6 +140,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.1.14 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect @@ -149,6 +150,7 @@ require ( github.com/shirou/gopsutil/v4 v4.24.9 // indirect github.com/shomali11/util v0.0.0-20180607005212-e0f70fd665ff // indirect github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sourcegraph/jsonrpc2 v0.2.0 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.9.0 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect diff --git a/go.sum b/go.sum index cbd2831e8..ce9401753 100644 --- a/go.sum +++ b/go.sum @@ -97,18 +97,22 @@ github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241017152835-2c30f563ab46 h1:8wZtnWSIYNV7DwD0Jr4HsbcRgezOrgDJ2Q0w9ABieKc= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241017152835-2c30f563ab46/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241017153232-1a62cecf1a61 h1:GSIhsdo/YXuZXI4q8xA8IrdOkkjfFp6O+DiNywk8s8U= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241017153232-1a62cecf1a61/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= github.com/formancehq/go-libs/v2 v2.0.1-0.20241022185745-110c95803b63 h1:DN6gDFwh3zO9VwV6Nt2tj4/BEecyfWfOdHp1YYJ5sBA= github.com/formancehq/go-libs/v2 v2.0.1-0.20241022185745-110c95803b63/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= -github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 h1:LOd5hxnXDIBcehFrpW1OnXk+VSs0yJXeu1iAOO+Hji4= -github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417/go.mod h1:btuSv05cYwi9BvLRxVs5zrunU+O1vTgigG1T6UsawcY= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163308-e06611e301d6 h1:on0q5s/WngVsTEllmADLdiAWwyG7qVAOzuqSBVs/mng= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163308-e06611e301d6/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163904-e440de7907c7 h1:x8vIRM5+y01pLs2YqnYcoUsvFJ/6cP5qDtsM248OmWM= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163904-e440de7907c7/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= +github.com/formancehq/numscript v0.0.8 h1:YAHcgtGxd6DWqwwiFaxWRVuXuBaW5HFx+GRuyUidlxU= +github.com/formancehq/numscript v0.0.8/go.mod h1:P8qnq15PyWUuhskZdsrzTTqVS5CBkhakDKEPAyF4oYQ= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/gkampitakis/ciinfo v0.3.0 h1:gWZlOC2+RYYttL0hBqcoQhM7h1qNkVqvRCV1fOvpAv8= -github.com/gkampitakis/ciinfo v0.3.0/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= -github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= -github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= -github.com/gkampitakis/go-snaps v0.5.4 h1:GX+dkKmVsRenz7SoTbdIEL4KQARZctkMiZ8ZKprRwT8= -github.com/gkampitakis/go-snaps v0.5.4/go.mod h1:ZABkO14uCuVxBHAXAfKG+bqNz+aa1bGPAg8jkI0Nk8Y= +github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= +github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= @@ -157,6 +161,7 @@ github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+ github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -222,8 +227,6 @@ github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMD github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= -github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -292,6 +295,8 @@ github.com/shomali11/xsql v0.0.0-20190608141458-bf76292144df h1:SVCDTuzM3KEk8WBw github.com/shomali11/xsql v0.0.0-20190608141458-bf76292144df/go.mod h1:K8jR5lDI2MGs9Ky+X2jIF4MwIslI0L8o8ijIlEq7/Vw= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/jsonrpc2 v0.2.0 h1:KjN/dC4fP6aN9030MZCJs9WQbTOjWHhrtKVpzzSrr/U= +github.com/sourcegraph/jsonrpc2 v0.2.0/go.mod h1:ZafdZgk/axhT1cvZAPOhw+95nz2I/Ra5qMlU4gTRwIo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -307,14 +312,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= -github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo= @@ -401,8 +398,8 @@ go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeX go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= -go.uber.org/fx v1.23.0 h1:lIr/gYWQGfTwGcSXWXu4vP5Ws6iqnNEIY+F/aFzCKTg= -go.uber.org/fx v1.23.0/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= +go.uber.org/fx v1.22.2 h1:iPW+OPxv0G8w75OemJ1RAnTUrF55zOJlXlo1TbJ0Buw= +go.uber.org/fx v1.22.2/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= diff --git a/internal/storage/driver/driver.go b/internal/storage/driver/driver.go index 41242f328..f2430cfb5 100644 --- a/internal/storage/driver/driver.go +++ b/internal/storage/driver/driver.go @@ -3,6 +3,7 @@ package driver import ( "context" "database/sql" + "errors" "fmt" "github.com/formancehq/go-libs/v2/collectionutils" "github.com/formancehq/go-libs/v2/metadata" @@ -128,10 +129,49 @@ func (d *Driver) OpenLedger(ctx context.Context, name string) (*ledgerstore.Stor func (d *Driver) Initialize(ctx context.Context) error { logging.FromContext(ctx).Debugf("Initialize driver") - err := Migrate(ctx, d.db) + err := d.detectRollbacks(ctx) + if err != nil { + return fmt.Errorf("detecting rollbacks: %w", err) + } + + err = Migrate(ctx, d.db) if err != nil { return fmt.Errorf("migrating system store: %w", err) } + + return nil +} + +func (d *Driver) detectRollbacks(ctx context.Context) error { + + logging.FromContext(ctx).Debugf("Checking for downgrades on system schema") + if err := detectDowngrades(GetMigrator(), ctx, d.db); err != nil { + return fmt.Errorf("detecting rollbacks of system schema: %w", err) + } + + type row struct { + Bucket string `bun:"bucket"` + } + rows := make([]row, 0) + if err := d.db.NewSelect(). + DistinctOn("bucket"). + ModelTableExpr("_system.ledgers"). + Column("bucket"). + Scan(ctx, &rows); err != nil { + err = postgres.ResolveError(err) + if errors.Is(err, postgres.ErrMissingTable) { + return nil + } + return err + } + + for _, r := range rows { + logging.FromContext(ctx).Debugf("Checking for downgrades on bucket '%s'", r.Bucket) + if err := detectDowngrades(bucket.GetMigrator(r.Bucket), ctx, d.db); err != nil { + return fmt.Errorf("detecting rollbacks on bucket '%s': %w", r.Bucket, err) + } + } + return nil } diff --git a/internal/storage/driver/errors.go b/internal/storage/driver/errors.go new file mode 100644 index 000000000..65452fb29 --- /dev/null +++ b/internal/storage/driver/errors.go @@ -0,0 +1,24 @@ +package driver + +import "fmt" + +type ErrRollbackDetected struct { + LastRegisterVersion int + LastAvailableVersion int +} + +func (e ErrRollbackDetected) Error() string { + return fmt.Sprintf("rollback detected, last register version: %d, last available version: %d", e.LastRegisterVersion, e.LastAvailableVersion) +} + +func (e ErrRollbackDetected) Is(err error) bool { + _, ok := err.(ErrRollbackDetected) + return ok +} + +func newErrRollbackDetected(lastRegisterVersion, lastAvailableVersion int) ErrRollbackDetected { + return ErrRollbackDetected{ + LastRegisterVersion: lastRegisterVersion, + LastAvailableVersion: lastAvailableVersion, + } +} diff --git a/internal/storage/driver/migrations.go b/internal/storage/driver/migrations.go index 340ab4a85..dfce0341a 100644 --- a/internal/storage/driver/migrations.go +++ b/internal/storage/driver/migrations.go @@ -2,6 +2,8 @@ package driver import ( "context" + "errors" + "fmt" "github.com/formancehq/go-libs/v2/time" "github.com/formancehq/go-libs/v2/platform/postgres" @@ -182,6 +184,27 @@ func Migrate(ctx context.Context, db bun.IDB) error { return GetMigrator().Up(ctx, db) } +func detectDowngrades(migrator *migrations.Migrator, ctx context.Context, db bun.IDB) error { + lastVersion, err := migrator.GetDBVersion(ctx, db) + if err != nil { + if !errors.Is(err, migrations.ErrMissingVersionTable) { + return fmt.Errorf("failed to get last version: %w", err) + } + } + if err == nil && lastVersion != -1 { + allMigrations, err := migrator.GetMigrations(ctx, db) + if err != nil { + return fmt.Errorf("failed to get all migrations: %w", err) + } + + if len(allMigrations) < int(lastVersion) { + return newErrRollbackDetected(int(lastVersion), len(allMigrations)) + } + } + + return nil +} + const aggregateObjects = ` create or replace function public.jsonb_concat(a jsonb, b jsonb) returns jsonb as 'select $1 || $2' diff --git a/openapi.yaml b/openapi.yaml index 4a7d602fe..2766718c9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -3478,9 +3478,7 @@ components: type: object properties: version: - type: integer - format: int64 - minimum: 0 + type: string example: 11 name: type: string diff --git a/openapi/v2.yaml b/openapi/v2.yaml index 3e2ed140d..cb4b4f02f 100644 --- a/openapi/v2.yaml +++ b/openapi/v2.yaml @@ -1748,9 +1748,7 @@ components: type: object properties: version: - type: integer - format: int64 - minimum: 0 + type: string example: 11 name: type: string diff --git a/pkg/client/.speakeasy/gen.lock b/pkg/client/.speakeasy/gen.lock index 3ac7e688a..dbd7dad59 100644 --- a/pkg/client/.speakeasy/gen.lock +++ b/pkg/client/.speakeasy/gen.lock @@ -1,12 +1,12 @@ lockVersion: 2.0.0 id: a9ac79e1-e429-4ee3-96c4-ec973f19bec3 management: - docChecksum: 043615f02d6da6be8aa8e5c006587e66 + docChecksum: db305ef0d86f319f00b70be811eefeea docVersion: v1 speakeasyVersion: 1.351.0 generationVersion: 2.384.1 - releaseVersion: 0.4.16 - configChecksum: 87ddc5f3dd8be9290d0bf5cabd849a48 + releaseVersion: 0.4.17 + configChecksum: 4844480c9de6ace6cb6abaa8ed6c2fe3 features: go: additionalDependencies: 0.1.0 diff --git a/pkg/client/.speakeasy/gen.yaml b/pkg/client/.speakeasy/gen.yaml index 50198e28b..1a4f05af2 100644 --- a/pkg/client/.speakeasy/gen.yaml +++ b/pkg/client/.speakeasy/gen.yaml @@ -15,7 +15,7 @@ generation: auth: oAuth2ClientCredentialsEnabled: true go: - version: 0.4.16 + version: 0.4.17 additionalDependencies: {} allowUnknownFieldsInWeakUnions: false clientServerStatusCodesAsErrors: true diff --git a/pkg/client/docs/models/components/v2migrationinfo.md b/pkg/client/docs/models/components/v2migrationinfo.md index 199094883..7b51f360c 100644 --- a/pkg/client/docs/models/components/v2migrationinfo.md +++ b/pkg/client/docs/models/components/v2migrationinfo.md @@ -5,7 +5,7 @@ | Field | Type | Required | Description | Example | | ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | -| `Version` | **int64* | :heavy_minus_sign: | N/A | 11 | +| `Version` | **string* | :heavy_minus_sign: | N/A | 11 | | `Name` | **string* | :heavy_minus_sign: | N/A | migrations:001 | | `Date` | [*time.Time](https://pkg.go.dev/time#Time) | :heavy_minus_sign: | N/A | | | `State` | [*components.V2MigrationInfoState](../../models/components/v2migrationinfostate.md) | :heavy_minus_sign: | N/A | | \ No newline at end of file diff --git a/pkg/client/formance.go b/pkg/client/formance.go index 949d62ce8..e03e73b19 100644 --- a/pkg/client/formance.go +++ b/pkg/client/formance.go @@ -143,9 +143,9 @@ func New(opts ...SDKOption) *Formance { sdkConfiguration: sdkConfiguration{ Language: "go", OpenAPIDocVersion: "v1", - SDKVersion: "0.4.16", + SDKVersion: "0.4.17", GenVersion: "2.384.1", - UserAgent: "speakeasy-sdk/go 0.4.16 2.384.1 v1 github.com/formancehq/stack/ledger/client", + UserAgent: "speakeasy-sdk/go 0.4.17 2.384.1 v1 github.com/formancehq/stack/ledger/client", Hooks: hooks.New(), }, } diff --git a/pkg/client/models/components/v2migrationinfo.go b/pkg/client/models/components/v2migrationinfo.go index 07a0a6869..1c8e962c6 100644 --- a/pkg/client/models/components/v2migrationinfo.go +++ b/pkg/client/models/components/v2migrationinfo.go @@ -36,7 +36,7 @@ func (e *V2MigrationInfoState) UnmarshalJSON(data []byte) error { } type V2MigrationInfo struct { - Version *int64 `json:"version,omitempty"` + Version *string `json:"version,omitempty"` Name *string `json:"name,omitempty"` Date *time.Time `json:"date,omitempty"` State *V2MigrationInfoState `json:"state,omitempty"` @@ -53,7 +53,7 @@ func (v *V2MigrationInfo) UnmarshalJSON(data []byte) error { return nil } -func (o *V2MigrationInfo) GetVersion() *int64 { +func (o *V2MigrationInfo) GetVersion() *string { if o == nil { return nil } diff --git a/pkg/testserver/api.go b/pkg/testserver/api.go index 305949de6..1e1377d1e 100644 --- a/pkg/testserver/api.go +++ b/pkg/testserver/api.go @@ -30,6 +30,10 @@ func GetInfo(ctx context.Context, srv *Server) (*operations.V2GetInfoResponse, e return srv.Client().Ledger.GetInfo(ctx) } +func GetLedgerInfo(ctx context.Context, srv *Server, request operations.V2GetLedgerInfoRequest) (*operations.V2GetLedgerInfoResponse, error) { + return srv.Client().Ledger.V2.GetLedgerInfo(ctx, request) +} + func CreateTransaction(ctx context.Context, srv *Server, request operations.V2CreateTransactionRequest) (*components.V2Transaction, error) { response, err := srv.Client().Ledger.V2.CreateTransaction(ctx, request) diff --git a/pkg/testserver/helpers.go b/pkg/testserver/helpers.go index 1e1aa5167..0966f63a0 100644 --- a/pkg/testserver/helpers.go +++ b/pkg/testserver/helpers.go @@ -6,7 +6,10 @@ import ( "github.com/formancehq/go-libs/v2/time" "github.com/formancehq/ledger/internal" "github.com/formancehq/stack/ledger/client/models/components" + "github.com/nats-io/nats.go" . "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/require" + "github.com/uptrace/bun" ) func NewTestServer(configurationProvider func() Configuration) *Deferred[*Server] { @@ -63,3 +66,23 @@ func ConvertSDKPostingToCorePosting(p components.V2Posting) ledger.Posting { Amount: p.Amount, } } + +func Subscribe(t T, testServer *Server) chan *nats.Msg { + subscription, ch, err := testServer.Subscribe() + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, subscription.Unsubscribe()) + }) + + return ch +} + +func ConnectToDatabase(t T, testServer *Server) *bun.DB { + db, err := testServer.Database() + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + return db +} \ No newline at end of file diff --git a/pkg/testserver/server.go b/pkg/testserver/server.go index c70670e81..857e90269 100644 --- a/pkg/testserver/server.go +++ b/pkg/testserver/server.go @@ -2,6 +2,7 @@ package testserver import ( "context" + "errors" "fmt" "io" "net/http" @@ -48,9 +49,13 @@ type Configuration struct { ExperimentalNumscriptRewrite bool } +type Logger interface { + Logf(fmt string, args ...any) +} + type Server struct { configuration Configuration - t T + logger Logger httpClient *ledgerclient.Formance cancel func() ctx context.Context @@ -58,9 +63,7 @@ type Server struct { id string } -func (s *Server) Start() { - s.t.Helper() - +func (s *Server) Start() error { rootCmd := cmd.NewRootCommand() args := []string{ "serve", @@ -174,7 +177,7 @@ func (s *Server) Start() { args = append(args, "--"+service.DebugFlag) } - s.t.Logf("Starting application with flags: %s", strings.Join(args, " ")) + s.logger.Logf("Starting application with flags: %s", strings.Join(args, " ")) rootCmd.SetArgs(args) rootCmd.SilenceErrors = true output := s.configuration.Output @@ -184,26 +187,28 @@ func (s *Server) Start() { rootCmd.SetOut(output) rootCmd.SetErr(output) - s.ctx = logging.TestingContext() - s.ctx, s.cancel = context.WithCancel(s.ctx) - s.ctx = service.ContextWithLifecycle(s.ctx) - s.ctx = httpserver.ContextWithServerInfo(s.ctx) + ctx := logging.TestingContext() + ctx = service.ContextWithLifecycle(ctx) + ctx = httpserver.ContextWithServerInfo(ctx) + ctx, cancel := context.WithCancel(ctx) - s.errorChan = make(chan error, 1) go func() { - s.errorChan <- rootCmd.ExecuteContext(s.ctx) + s.errorChan <- rootCmd.ExecuteContext(ctx) }() select { - case <-service.Ready(s.ctx): + case <-service.Ready(ctx): case err := <-s.errorChan: + cancel() if err != nil { - require.NoError(s.t, err) - } else { - require.Fail(s.t, "unexpected service stop") + return err } + + return errors.New("unexpected service stop") } + s.ctx, s.cancel = ctx, cancel + var transport http.RoundTripper = &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -219,13 +224,13 @@ func (s *Server) Start() { Transport: transport, }), ) -} -func (s *Server) Stop(ctx context.Context) { - s.t.Helper() + return nil +} +func (s *Server) Stop(ctx context.Context) error { if s.cancel == nil { - return + return nil } s.cancel() s.cancel = nil @@ -234,15 +239,15 @@ func (s *Server) Stop(ctx context.Context) { select { case <-service.Stopped(s.ctx): case <-ctx.Done(): - require.Fail(s.t, "service should have been stopped") + return errors.New("service should have been stopped") } // Ensure the app has been properly shutdown select { case err := <-s.errorChan: - require.NoError(s.t, err) + return err case <-ctx.Done(): - require.Fail(s.t, "service should have been stopped without error") + return errors.New("service should have been stopped without error") } } @@ -250,40 +255,45 @@ func (s *Server) Client() *ledgerclient.Formance { return s.httpClient } -func (s *Server) Restart(ctx context.Context) { - s.t.Helper() +func (s *Server) Restart(ctx context.Context) error { + if err := s.Stop(ctx); err != nil { + return err + } + if err := s.Start(); err != nil { + return err + } - s.Stop(ctx) - s.Start() + return nil } -func (s *Server) Database() *bun.DB { +func (s *Server) Database() (*bun.DB, error) { db, err := bunconnect.OpenSQLDB(s.ctx, s.configuration.PostgresConfiguration) - require.NoError(s.t, err) - s.t.Cleanup(func() { - require.NoError(s.t, db.Close()) - }) + if err != nil { + return nil, err + } - return db + return db, nil } -func (s *Server) Subscribe() chan *nats.Msg { +func (s *Server) Subscribe() (*nats.Subscription, chan *nats.Msg, error) { if s.configuration.NatsURL == "" { - require.Fail(s.t, "NATS URL must be set") + return nil, nil, errors.New("NATS URL must be set") } ret := make(chan *nats.Msg) conn, err := nats.Connect(s.configuration.NatsURL) - require.NoError(s.t, err) + if err != nil { + return nil, nil, err + } subscription, err := conn.Subscribe(s.id, func(msg *nats.Msg) { ret <- msg }) - require.NoError(s.t, err) - s.t.Cleanup(func() { - require.NoError(s.t, subscription.Unsubscribe()) - }) - return ret + if err != nil { + return nil, nil, err + } + + return subscription, ret, nil } func (s *Server) URL() string { @@ -294,18 +304,19 @@ func New(t T, configuration Configuration) *Server { t.Helper() srv := &Server{ - t: t, + logger: t, configuration: configuration, id: uuid.NewString()[:8], + errorChan: make(chan error, 1), } t.Logf("Start testing server") - srv.Start() + require.NoError(t, srv.Start()) t.Cleanup(func() { t.Logf("Stop testing server") ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - srv.Stop(ctx) + require.NoError(t, srv.Stop(ctx)) }) return srv diff --git a/test/e2e/api_accounts_metadata_test.go b/test/e2e/api_accounts_metadata_test.go index a0e618afc..c6be424bf 100644 --- a/test/e2e/api_accounts_metadata_test.go +++ b/test/e2e/api_accounts_metadata_test.go @@ -29,7 +29,7 @@ var _ = Context("Ledger accounts list API tests", func() { }) var events chan *nats.Msg BeforeEach(func() { - events = testServer.GetValue().Subscribe() + events = Subscribe(GinkgoT(), testServer.GetValue()) }) BeforeEach(func() { diff --git a/test/e2e/api_ledgers_import_test.go b/test/e2e/api_ledgers_import_test.go index 9576a5127..31c421257 100644 --- a/test/e2e/api_ledgers_import_test.go +++ b/test/e2e/api_ledgers_import_test.go @@ -178,7 +178,7 @@ var _ = Context("Ledger engine tests", func() { // we take a lock on the ledgers table to force the process to wait // while we will make a concurrent request JustBeforeEach(func() { - db = testServer.GetValue().Database() + db = ConnectToDatabase(GinkgoT(), testServer.GetValue()) sqlTx, err = db.BeginTx(ctx, &sql.TxOptions{}) Expect(err).To(BeNil()) diff --git a/test/e2e/api_transactions_create_test.go b/test/e2e/api_transactions_create_test.go index 7512dcffb..71f927b43 100644 --- a/test/e2e/api_transactions_create_test.go +++ b/test/e2e/api_transactions_create_test.go @@ -66,7 +66,7 @@ var _ = Context("Ledger accounts list API tests", func() { err error ) BeforeEach(func() { - events = testServer.GetValue().Subscribe() + events = Subscribe(GinkgoT(), testServer.GetValue()) req = operations.V2CreateTransactionRequest{ V2PostTransaction: components.V2PostTransaction{ Timestamp: ×tamp, diff --git a/test/e2e/api_transactions_revert_test.go b/test/e2e/api_transactions_revert_test.go index d8d4e7028..5b51b3628 100644 --- a/test/e2e/api_transactions_revert_test.go +++ b/test/e2e/api_transactions_revert_test.go @@ -47,7 +47,7 @@ var _ = Context("Ledger accounts list API tests", func() { err error ) BeforeEach(func() { - events = testServer.GetValue().Subscribe() + events = Subscribe(GinkgoT(), testServer.GetValue()) tx, err = CreateTransaction( ctx, testServer.GetValue(), diff --git a/test/e2e/lifecycle_test.go b/test/e2e/lifecycle_test.go index 81908d9f4..968a5f69f 100644 --- a/test/e2e/lifecycle_test.go +++ b/test/e2e/lifecycle_test.go @@ -8,10 +8,12 @@ import ( "github.com/formancehq/go-libs/v2/logging" "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/go-libs/v2/time" + ledger "github.com/formancehq/ledger/internal" ledgerevents "github.com/formancehq/ledger/pkg/events" . "github.com/formancehq/ledger/pkg/testserver" "github.com/formancehq/stack/ledger/client/models/components" "github.com/formancehq/stack/ledger/client/models/operations" + "github.com/google/uuid" "github.com/nats-io/nats.go" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -35,7 +37,7 @@ var _ = Context("Ledger application lifecycle tests", func() { }) var events chan *nats.Msg BeforeEach(func() { - events = testServer.GetValue().Subscribe() + events = Subscribe(GinkgoT(), testServer.GetValue()) }) When("starting the service", func() { @@ -47,7 +49,7 @@ var _ = Context("Ledger application lifecycle tests", func() { }) When("restarting the service", func() { BeforeEach(func(ctx context.Context) { - testServer.GetValue().Restart(ctx) + Expect(testServer.GetValue().Restart(ctx)).To(BeNil()) }) It("should be ok", func() {}) }) @@ -66,7 +68,7 @@ var _ = Context("Ledger application lifecycle tests", func() { // lock logs table to block transactions creation requests // the first tx will block on the log insertion // the next transaction will block earlier on advisory lock acquirement for accounts - db := testServer.GetValue().Database() + db := ConnectToDatabase(GinkgoT(), testServer.GetValue()) sqlTx, err = db.BeginTx(ctx, &sql.TxOptions{}) Expect(err).To(BeNil()) DeferCleanup(func() { @@ -126,7 +128,7 @@ var _ = Context("Ledger application lifecycle tests", func() { ctx, cancel := context.WithTimeout(ctx, serverRestartTimeout) DeferCleanup(cancel) - testServer.GetValue().Restart(ctx) + Expect(testServer.GetValue().Restart(ctx)).To(BeNil()) }) }() @@ -162,3 +164,51 @@ var _ = Context("Ledger application lifecycle tests", func() { }) }) }) + +var _ = Context("Ledger downgrade tests", func() { + var ( + db = UseTemplatedDatabase() + ctx = logging.TestingContext() + ) + + testServer := NewTestServer(func() Configuration { + return Configuration{ + PostgresConfiguration: db.GetValue().ConnectionOptions(), + Output: GinkgoWriter, + Debug: debug, + NatsURL: natsServer.GetValue().ClientURL(), + } + }) + + When("inserting new migrations into the database", func() { + BeforeEach(func() { + ledgerName := uuid.NewString()[:8] + + err := CreateLedger(ctx, testServer.GetValue(), operations.V2CreateLedgerRequest{ + Ledger: ledgerName, + }) + Expect(err).To(BeNil()) + + info, err := GetLedgerInfo(ctx, testServer.GetValue(), operations.V2GetLedgerInfoRequest{ + Ledger: ledgerName, + }) + Expect(err).To(BeNil()) + + // Insert a fake migration into the database to simulate a downgrade + _, err = ConnectToDatabase(GinkgoT(), testServer.GetValue()). + NewInsert(). + ModelTableExpr(ledger.DefaultBucket + ".goose_db_version"). + Model(&map[string]any{ + "version_id": len(info.V2LedgerInfoResponse.Data.Storage.Migrations) + 1, + "is_applied": true, + }). + Exec(ctx) + Expect(err).To(BeNil()) + }) + Context("then when restarting the service", func() { + It("Should fail", func() { + Expect(testServer.GetValue().Restart(ctx)).NotTo(BeNil()) + }) + }) + }) +}) diff --git a/test/performance/env_testserver_test.go b/test/performance/env_testserver_test.go index 5f05ef75c..e39931737 100644 --- a/test/performance/env_testserver_test.go +++ b/test/performance/env_testserver_test.go @@ -34,8 +34,7 @@ func (e *TestServerEnv) URL() string { } func (e *TestServerEnv) Stop(ctx context.Context) error { - e.testServer.Stop(ctx) - return nil + return e.testServer.Stop(ctx) } var _ Env = (*TestServerEnv)(nil)