feat: remove sqlite from ledger (#109)

formancehq · May 12, 2023 · 82e60e5 · 82e60e5
1 parent ed3d108
commit 82e60e5
Show file tree

Hide file tree

Showing 32 changed files with 262 additions and 799 deletions.
diff --git a/Taskfile.yaml b/Taskfile.yaml
@@ -37,25 +37,12 @@ tasks:
         go test {{.TAGS}} {{if eq .VERBOSE "true"}}-v{{end}} -coverpkg {{.PKG}}
         -coverprofile coverage.out -covermode atomic {{.PKG}}
     env:
-      STORAGE_DRIVER: "postgres"
       STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:[email protected]/ledger"
 
   tests:local:
     cmds:
-      - task: tests:local:sqlite
       - task: tests:local:postgres
 
-  tests:local:sqlite:
-    cmds:
-      - >
-        go test {{.TAGS}} {{if eq .VERBOSE "true"}}-v{{end}} {{.FAILFAST}} -coverpkg {{.PKG}}
-        -coverprofile coverage.out -covermode atomic
-        -run={{.RUN}} -timeout {{.TIMEOUT}} {{.PKG}} |
-        sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' |
-        sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' |
-        sed ''/RUN/s//$(printf "\033[34mRUN\033[0m")/''
-      - task: print:coverage
-
   tests:local:postgres:
     deps: [postgres]
     cmds:
@@ -68,7 +55,6 @@ tasks:
         sed ''/RUN/s//$(printf "\033[34mRUN\033[0m")/''
       - task: print:coverage
     env:
-      STORAGE_DRIVER: "postgres"
       STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:[email protected]/ledger"
 
   print:coverage:
@@ -91,7 +77,6 @@ tasks:
         | tee {{.BENCH_RESULTS_FILE}}
       - benchstat {{.BENCH_RESULTS_FILE}}
     env:
-      STORAGE_DRIVER: "postgres"
       STORAGE_POSTGRES_CONN_STRING: "postgresql://ledger:[email protected]/ledger"
 
   bench:cpu:

diff --git a/cmd/container.go b/cmd/container.go
@@ -16,6 +16,7 @@ import (
 	"github.com/formancehq/stack/libs/go-libs/otlp/otlptraces"
 	"github.com/formancehq/stack/libs/go-libs/publish"
 	"github.com/formancehq/stack/libs/go-libs/service"
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
 	"github.com/spf13/viper"
 	"go.uber.org/fx"
@@ -29,7 +30,7 @@ func resolveOptions(v *viper.Viper, userOptions ...fx.Option) []fx.Option {
 
 	debug := v.GetBool(service.DebugFlag)
 	if debug {
-		sqlstorage.InstrumentalizeSQLDrivers()
+		sqlstorage.InstrumentalizeSQLDriver()
 	}
 
 	options = append(options, publish.CLIPublisherModule(v, ServiceName), bus.LedgerMonitorModule())
@@ -56,26 +57,12 @@ func resolveOptions(v *viper.Viper, userOptions ...fx.Option) []fx.Option {
 
 	// Handle api part
 	options = append(options, api.Module(api.Config{
-		StorageDriver: v.GetString(storageDriverFlag),
-		Version:       Version,
+		Version: Version,
 	}))
 
 	// Handle storage driver
 	options = append(options, sqlstorage.DriverModule(sqlstorage.ModuleConfig{
-		StorageDriver: v.GetString(storageDriverFlag),
-		SQLiteConfig: func() *sqlstorage.SQLiteConfig {
-			if v.GetString(storageDriverFlag) != sqlstorage.SQLite.String() {
-				return nil
-			}
-			return &sqlstorage.SQLiteConfig{
-				Dir:    v.GetString(storageDirFlag),
-				DBName: v.GetString(storageSQLiteDBNameFlag),
-			}
-		}(),
 		PostgresConfig: func() *sqlstorage.PostgresConfig {
-			if v.GetString(storageDriverFlag) != sqlstorage.PostgreSQL.String() {
-				return nil
-			}
 			return &sqlstorage.PostgresConfig{
 				ConnString: v.GetString(storagePostgresConnectionStringFlag),
 			}
@@ -116,6 +103,7 @@ func resolveOptions(v *viper.Viper, userOptions ...fx.Option) []fx.Option {
 			})
 		})
 		res = append(res, middlewares.Log())
+		res = append(res, middleware.Recoverer)
 		return res
 	}))
 

diff --git a/cmd/container_test.go b/cmd/container_test.go
@@ -17,7 +17,6 @@ import (
 	"github.com/formancehq/ledger/pkg/ledger"
 	"github.com/formancehq/ledger/pkg/redis"
 	"github.com/formancehq/ledger/pkg/storage"
-	"github.com/formancehq/ledger/pkg/storage/sqlstorage"
 	"github.com/formancehq/stack/libs/go-libs/logging"
 	"github.com/formancehq/stack/libs/go-libs/otlp/otlptraces"
 	"github.com/formancehq/stack/libs/go-libs/pgtesting"
@@ -39,16 +38,9 @@ func TestContainers(t *testing.T) {
 	}
 
 	for _, tc := range []testCase{
-		{
-			name: "default",
-			init: func(v *viper.Viper) {
-				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-			},
-		},
 		{
 			name: "default-with-opentelemetry-traces-on-stdout",
 			init: func(v *viper.Viper) {
-				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
 				v.Set(otlptraces.OtelTracesFlag, true)
 				v.Set(otlptraces.OtelTracesExporterFlag, "stdout")
 			},
@@ -83,7 +75,6 @@ func TestContainers(t *testing.T) {
 		{
 			name: "default-with-opentelemetry-traces-on-stdout-and-batch",
 			init: func(v *viper.Viper) {
-				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
 				v.Set(otlptraces.OtelTracesFlag, true)
 				v.Set(otlptraces.OtelTracesExporterFlag, "stdout")
 				v.Set(otlptraces.OtelTracesBatchFlag, true)
@@ -121,23 +112,20 @@ func TestContainers(t *testing.T) {
 		{
 			name: "default-with-opentelemetry-traces-on-otlp",
 			init: func(v *viper.Viper) {
-				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
 				v.Set(otlptraces.OtelTracesFlag, true)
 				v.Set(otlptraces.OtelTracesExporterFlag, "otlp")
 			},
 		},
 		{
 			name: "default-with-opentelemetry-traces-on-jaeger",
 			init: func(v *viper.Viper) {
-				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
 				v.Set(otlptraces.OtelTracesFlag, true)
 				v.Set(otlptraces.OtelTracesExporterFlag, "jaeger")
 			},
 		},
 		{
 			name: "pg",
 			init: func(v *viper.Viper) {
-				v.Set(storageDriverFlag, sqlstorage.PostgreSQL.String())
 				v.Set(storagePostgresConnectionStringFlag, db.ConnString())
 			},
 			options: []fx.Option{
@@ -243,11 +231,9 @@ func TestContainers(t *testing.T) {
 			)
 			v := viper.New()
 			// Default options
-			v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-			v.Set(storageDirFlag, "/tmp")
+			v.Set(storagePostgresConnectionStringFlag, db.ConnString())
 			v.Set(cacheCapacityBytes, 100000000)
 			v.Set(cacheMaxNumKeys, 100)
-			//v.Set(storageSQLiteDBNameFlag, uuid.New())
 			tc.init(v)
 			app := NewContainer(v, options...)
 

diff --git a/cmd/internal/analytics_test.go b/cmd/internal/analytics_test.go
@@ -3,15 +3,14 @@ package internal
 import (
 	"context"
 	"net/http"
-	"os"
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/formancehq/ledger/pkg/ledger"
+	"github.com/formancehq/ledger/pkg/ledgertesting"
 	"github.com/formancehq/ledger/pkg/storage"
 	"github.com/formancehq/ledger/pkg/storage/sqlstorage"
-	"github.com/pborman/uuid"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
@@ -96,10 +95,16 @@ func TestAnalyticsModule(t *testing.T) {
 	app := fx.New(
 		module,
 		fx.Provide(func(lc fx.Lifecycle) (storage.Driver[ledger.Store], error) {
-			id := uuid.New()
-			driver := sqlstorage.NewDriver("sqlite", sqlstorage.NewSQLiteDB(os.TempDir(), id))
+			driver, stopFn, err := ledgertesting.StorageDriver()
+			if err != nil {
+				return nil, err
+			}
 			lc.Append(fx.Hook{
 				OnStart: driver.Initialize,
+				OnStop: func(ctx context.Context) error {
+					stopFn()
+					return driver.Close(ctx)
+				},
 			})
 			return sqlstorage.NewLedgerStorageDriverFromRawDriver(driver), nil
 		}),

diff --git a/cmd/root.go b/cmd/root.go
@@ -3,26 +3,18 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"path"
 
 	"github.com/formancehq/ledger/cmd/internal"
 	"github.com/formancehq/ledger/pkg/redis"
 	_ "github.com/formancehq/ledger/pkg/storage/sqlstorage/migrates/9-add-pre-post-volumes"
 	"github.com/formancehq/stack/libs/go-libs/otlp/otlptraces"
 	"github.com/formancehq/stack/libs/go-libs/publish"
 	"github.com/formancehq/stack/libs/go-libs/service"
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 const (
-	// Deprecated
-	storageDirFlag = "storage.dir"
-	// Deprecated
-	storageDriverFlag = "storage.driver"
-	// Deprecated
-	storageSQLiteDBNameFlag             = "storage.sqlite.db_name"
 	storagePostgresConnectionStringFlag = "storage.postgres.conn_string"
 	bindFlag                            = "bind"
 	lockStrategyFlag                    = "lock-strategy"
@@ -52,17 +44,6 @@ func NewRootCommand() *cobra.Command {
 		Use:               "ledger",
 		Short:             "ledger",
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if viper.GetString(storageDriverFlag) == "sqlite" {
-				_, _ = fmt.Fprintln(os.Stderr,
-					"WARNING: SQLite is being deprecated and will not be supported starting from the v2 of the Formance Ledger. Please use Postgres instead.")
-			}
-			err := os.MkdirAll(viper.GetString(storageDirFlag), 0700)
-			if err != nil {
-				return errors.Wrap(err, "creating storage directory")
-			}
-			return nil
-		},
 	}
 
 	serve := NewServe()
@@ -84,17 +65,9 @@ func NewRootCommand() *cobra.Command {
 
 	root.AddCommand(NewDocCommand())
 
-	home, err := os.UserHomeDir()
-	if err != nil {
-		home = "/root"
-	}
-
 	root.PersistentFlags().Bool(service.DebugFlag, false, "Debug mode")
-	root.PersistentFlags().String(storageDriverFlag, "sqlite", "Storage driver")
-	root.PersistentFlags().String(storageDirFlag, path.Join(home, ".formance/data"), "Storage directory (for sqlite)")
-	root.PersistentFlags().String(storageSQLiteDBNameFlag, "formance", "SQLite database name")
 	root.PersistentFlags().String(storagePostgresConnectionStringFlag, "postgresql://localhost/postgres", "Postgre connection string")
-	root.PersistentFlags().String(bindFlag, "localhost:3068", "API bind address")
+	root.PersistentFlags().String(bindFlag, "0.0.0.0:3068", "API bind address")
 	root.PersistentFlags().String(lockStrategyFlag, "memory", "Lock strategy (memory, none, redis)")
 	root.PersistentFlags().String(lockStrategyRedisUrlFlag, "", "Redis url when using redis locking strategy")
 	root.PersistentFlags().Duration(lockStrategyRedisDurationFlag, redis.DefaultLockDuration, "Lock duration")
@@ -111,7 +84,7 @@ func NewRootCommand() *cobra.Command {
 	internal.InitAnalyticsFlags(root, DefaultSegmentWriteKey)
 	publish.InitCLIFlags(root)
 
-	if err = viper.BindPFlags(root.PersistentFlags()); err != nil {
+	if err := viper.BindPFlags(root.PersistentFlags()); err != nil {
 		panic(err)
 	}
 

diff --git a/cmd/root_test.go b/cmd/root_test.go
@@ -42,7 +42,7 @@ func TestServer(t *testing.T) {
 		},
 		{
 			name: "pg",
-			args: []string{"--storage.driver", "postgres", "--storage.postgres.conn_string", db.ConnString()},
+			args: []string{"--storage.postgres.conn_string", db.ConnString()},
 		},
 		{
 			name: "pg-with-env-var",