Merge pull request #341 from coinbase/jingwei/all-in-memory

[cli]enable allinmemory in cli

cmd/root.go

@@ -55,6 +55,8 @@ var (
 	dataResultFile         string
 	constructionResultFile string
 	dataDirectory          string
+	inMemoryMode           bool
+	tableSize              int64
 
 	// Config is the populated *configuration.Configuration from
 	// the configurationFile. If none is provided, this is set
@@ -263,6 +265,21 @@ default values.`,
 		"",
 		"Data-dir configures the location of logs and data for validation. This will override the data_directory from configuration file",
 	)
+
+	checkDataCmd.Flags().Int64Var(
+		&tableSize,
+		"table-size",
+		0,
+		"Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file",
+	)
+
+	checkDataCmd.Flags().BoolVar(
+		&inMemoryMode,
+		"in-memory-mode",
+		false,
+		"in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled",
+	)
+
 	rootCmd.AddCommand(checkDataCmd)
 	checkConstructionCmd.Flags().StringVar(
 		&asserterConfigurationFile,
@@ -372,6 +389,14 @@ func initConfig() {
 	if len(dataDirectory) != 0 {
 		Config.DataDirectory = dataDirectory
 	}
+
+	if inMemoryMode {
+		Config.AllInMemoryEnabled = inMemoryMode
+	}
+
+	if tableSize != 0 {
+		Config.TableSize = &tableSize
+	}
 }
 
 func ensureDataDirectoryExists() {

configuration/types.go

@@ -414,6 +414,21 @@ type Configuration struct {
 	// but can use 10s of GBs of RAM, even with pruning enabled.
 	MemoryLimitDisabled bool `json:"memory_limit_disabled"`
 
+	// AllInMemoryDisabled configures storage to increase memory
+	// usage. Enabling this massively increases performance
+	// but can use >20s of GBs of RAM, even with pruning enabled.
+	AllInMemoryEnabled bool `json:"all_in_memory_enabled"`
+
+	// TableSize unit is GB, enable users to define MaxTableSize
+	// when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config
+	// default value is 6, modification range is [2, 100]
+	TableSize *int64 `json:"table_size,omitempty"`
+
+	// TableSize unit is MB, enable users to define ValueLogFileSize
+	// when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config
+	// default value is 512, modification range is [128, 2048]
+	ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"`
+
 	// SeenBlockWorkers is the number of goroutines spawned to store
 	// seen blocks in storage before we attempt to sequence. If not populated,
 	// this value defaults to runtime.NumCPU().

examples/configuration/default.json

@@ -15,6 +15,7 @@
  "log_configuration": false,
  "compression_disabled": false,
  "memory_limit_disabled": false,
+ "all_in_memory_enabled": false,
  "error_stack_trace_disabled": false,
  "coin_supported": false,
  "construction": null,

go.mod

@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.16
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4
+	github.com/coinbase/rosetta-sdk-go v0.7.11
 	github.com/fatih/color v1.13.0
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect

go.sum

@@ -97,8 +97,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
 github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY=
 github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI=
-github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 h1:cHRCEzEk+4xerPtuJgO5a71qp7jIwGDKCKq+pte2VHA=
-github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM=
+github.com/coinbase/rosetta-sdk-go v0.7.11 h1:T7camDGstlX2ENVE0QHR1AVu3asSXED0vg1xDaF6bYE=
+github.com/coinbase/rosetta-sdk-go v0.7.11/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM=
 github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ=
 github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q=

pkg/tester/data.go

@@ -65,6 +65,18 @@ const (
 	// EndAtTipCheckInterval is the frequency that EndAtTip condition
 	// is evaludated
 	EndAtTipCheckInterval = 10 * time.Second
+
+	//MinTableSize unit is GB
+	MinTableSize = int64(2)
+
+	//MaxTableSize unit is GB
+	MaxTableSize = int64(100)
+
+	//MinTableSize unit is MB
+	MinValueLogFileSize = int64(128)
+
+	//MaxTableSize unit is MB
+	MaxValueLogFileSize = int64(2048)
 )
 
 var _ http.Handler = (*DataTester)(nil)
@@ -152,17 +164,52 @@ func InitializeData(
 	}
 
 	opts := []database.BadgerOption{}
-	if config.CompressionDisabled {
-		opts = append(opts, database.WithoutCompression())
-	}
-	if config.MemoryLimitDisabled {
+	dataPathBackup := dataPath
+
+	if config.AllInMemoryEnabled{
 		opts = append(
 			opts,
-			database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)),
+			database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)),
+			database.WithoutCompression(),
 		)
+		// for all in memory mode, the path need to be "", as badgerDB will not write to disk
+		dataPathBackup = ""
+	} else {
+		if config.CompressionDisabled {
+			opts = append(opts, database.WithoutCompression())
+		}
+		if config.MemoryLimitDisabled {
+			opts = append(
+				opts,
+				database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)),
+			)
+		}
+	}
+
+	// If we enable all-in-memory or L0-in-memory mode, badger DB's TableSize and ValueLogFileSize will change
+	// according to users config. tableSize means the LSM table size, when the table more than the tableSize, 
+	// will trigger a compact. 
+	// In default mode, we will not change the badger DB's TableSize and ValueLogFileSize for limiting memory usage
+	if config.AllInMemoryEnabled || config.MemoryLimitDisabled {
+		if(config.TableSize != nil) {
+			if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) {
+				opts = append(
+					opts,
+					database.WithTableSize(*config.TableSize),
+				)
+			}
+		}
+		if(config.ValueLogFileSize != nil) {
+			if(*config.TableSize >= MinValueLogFileSize && *config.TableSize <= MinValueLogFileSize) {
+				opts = append(
+					opts,
+					database.WithValueLogFileSize(*config.TableSize),
+				)
+			}
+		}
 	}
 
-	localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...)
+	localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("%s: unable to initialize database", err.Error())
 	}