From 75bf5b075ab7b404341924a7d22c69fc6601da18 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 2 Oct 2024 12:45:24 +0200 Subject: [PATCH] feat: add migration test using concrete database --- internal/storage/driver/driver.go | 20 +++++ test/migrations/README.md | 20 +++++ test/migrations/upgrade_test.go | 141 ++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 test/migrations/README.md create mode 100644 test/migrations/upgrade_test.go diff --git a/internal/storage/driver/driver.go b/internal/storage/driver/driver.go index 7aa8fd644..94ac993a4 100644 --- a/internal/storage/driver/driver.go +++ b/internal/storage/driver/driver.go @@ -240,6 +240,26 @@ func (d *Driver) UpgradeAllBuckets(ctx context.Context) error { return nil } +func (d *Driver) UpgradeAllLedgers(ctx context.Context) error { + err := bunpaginate.Iterate(ctx, ledgercontroller.NewListLedgersQuery(10), + func(ctx context.Context, q ledgercontroller.ListLedgersQuery) (*bunpaginate.Cursor[ledger.Ledger], error) { + return d.ListLedgers(ctx, q) + }, + func(cursor *bunpaginate.Cursor[ledger.Ledger]) error { + for _, ledger := range cursor.Data { + if err := ledgerstore.Migrate(ctx, d.db, ledger); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + + return nil +} + func New(db *bun.DB) *Driver { return &Driver{ db: db, diff --git a/test/migrations/README.md b/test/migrations/README.md new file mode 100644 index 000000000..95c4a621f --- /dev/null +++ b/test/migrations/README.md @@ -0,0 +1,20 @@ +# Migrations test + +This package allow to test the migration of an existing database regarding current code. + +The test can be run using the following command : +```shell +go test . \ + -databases.source +``` + +The test will start a new postgres server, copy the database inside, then apply migrations. + +Additionally, you can add the flag : +```shell +go test . \ + -databases.source \ + -databases.destination +``` + +In this case, the destination database will be used and no local postgres server will be started. diff --git a/test/migrations/upgrade_test.go b/test/migrations/upgrade_test.go new file mode 100644 index 000000000..ae95994e9 --- /dev/null +++ b/test/migrations/upgrade_test.go @@ -0,0 +1,141 @@ +package migrations + +import ( + "flag" + "fmt" + "github.com/formancehq/go-libs/bun/bunconnect" + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/go-libs/testing/docker" + "github.com/formancehq/go-libs/testing/platform/pgtesting" + "github.com/formancehq/ledger/internal/storage/driver" + "github.com/ory/dockertest/v3" + dockerlib "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/require" + "github.com/xo/dburl" + "os" + "strings" + "testing" +) + +var ( + sourceDatabase string + destinationDatabase string +) + +func TestMain(m *testing.M) { + flag.StringVar(&sourceDatabase, "databases.source", "", "Source database") + flag.StringVar(&destinationDatabase, "databases.destination", "", "Destination database") + flag.Parse() + + os.Exit(m.Run()) +} + +func TestMigrations(t *testing.T) { + if sourceDatabase == "" { + t.Skip() + } + // debug + //sourceDatabase = "postgresql://formance:formance@127.0.0.1:5432/qkjnwtkxwaof-oknl-ledger?sslmode=disable" + + ctx := logging.TestingContext() + dockerPool := docker.NewPool(t, logging.Testing()) + + if destinationDatabase == "" { + pgServer := pgtesting.CreatePostgresServer(t, dockerPool) + destinationDatabase = pgServer.GetDSN() + } + + copyDatabase(t, dockerPool, sourceDatabase, destinationDatabase) + + db, err := bunconnect.OpenSQLDB(ctx, bunconnect.ConnectionOptions{ + DatabaseSourceName: destinationDatabase, + }) + require.NoError(t, err) + + // Migrate database + driver := driver.New(db) + require.NoError(t, driver.Initialize(ctx)) + require.NoError(t, driver.UpgradeAllLedgers(ctx)) +} + +func copyDatabase(t *testing.T, dockerPool *docker.Pool, source, destination string) { + resource := dockerPool.Run(docker.Configuration{ + RunOptions: &dockertest.RunOptions{ + Repository: "postgres", + Tag: "15-alpine", + Entrypoint: []string{"sleep", "infinity"}, + }, + HostConfigOptions: []func(config *dockerlib.HostConfig){ + func(config *dockerlib.HostConfig) { + config.NetworkMode = "host" + }, + }, + }) + + execArgs := []string{"sh", "-c", fmt.Sprintf(` + %s | %s + `, + preparePGDumpCommand(t, source), + preparePSQLCommand(t, destination), + )} + + fmt.Printf("Exec command: %s\n", execArgs) + + _, err := resource.Exec(execArgs, dockertest.ExecOptions{ + StdOut: os.Stdout, + StdErr: os.Stdout, + }) + + require.NoError(t, err) +} + +func preparePGDumpCommand(t *testing.T, dsn string) string { + parsedSource, err := dburl.Parse(dsn) + require.NoError(t, err) + + args := make([]string, 0) + + password, ok := parsedSource.User.Password() + if ok { + args = append(args, "PGPASSWORD="+password) + } + + args = append(args, + "pg_dump", + "--no-owner", // skip roles + "-x", // Skip privileges + "-h", parsedSource.Hostname(), + "-p", parsedSource.Port(), + ) + + if username := parsedSource.User.Username(); username != "" { + args = append(args, "-U", username) + } + + return strings.Join(append(args, parsedSource.Path[1:]), " ") +} + +func preparePSQLCommand(t *testing.T, dsn string) string { + parsedSource, err := dburl.Parse(dsn) + require.NoError(t, err) + + args := make([]string, 0) + + password, ok := parsedSource.User.Password() + if ok { + args = append(args, "PGPASSWORD="+password) + } + + args = append(args, + "psql", + "-h", parsedSource.Hostname(), + "-p", parsedSource.Port(), + parsedSource.Path[1:], + ) + + if username := parsedSource.User.Username(); username != "" { + args = append(args, "-U", username) + } + + return strings.Join(args, " ") +}