From fdd7178dfdb4eb3d5a8080cd925b5a637f434f6b Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 8 Aug 2022 17:51:29 -0700 Subject: [PATCH 01/10] enable allinmemory in cli --- configuration/types.go | 15 +++++++++++++ go.mod | 2 ++ pkg/tester/data.go | 51 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 603afbef..3f3af83f 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -414,6 +414,21 @@ type Configuration struct { // but can use 10s of GBs of RAM, even with pruning enabled. MemoryLimitDisabled bool `json:"memory_limit_disabled"` + // AllInMemoryDisabled configures storage to increase memory + // usage. Enabling this massively increases performance + // but can use 20s of GBs of RAM, even with pruning enabled. + AllInMemoryEnabled bool `json:"all_in_memory_enabled"` + + // TableSize unit is GB, enable users to define MaxTableSize + // when AllInMemoryEnabled == true, Cli will look up this config + // default value is 6, modification range is [3, 100] + TableSize *int64 `json:"table_size,omitempty"` + + // TableSize unit is MB, enable users to define ValueLogFileSize + // when AllInMemoryEnabled == true, Cli will look up this config + // default value is 512, modification range is [256, 2048] + ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"` + // SeenBlockWorkers is the number of goroutines spawned to store // seen blocks in storage before we attempt to sequence. If not populated, // this value defaults to runtime.NumCPU(). diff --git a/go.mod b/go.mod index 4db9cbf4..ac5cbc7e 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/coinbase/rosetta-cli go 1.16 +replace github.com/coinbase/rosetta-sdk-go => /Users/jingweizhuang/crypto/rosetta-sdk-go + require ( github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 github.com/fatih/color v1.13.0 diff --git a/pkg/tester/data.go b/pkg/tester/data.go index e74e9bdc..23662230 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -65,6 +65,18 @@ const ( // EndAtTipCheckInterval is the frequency that EndAtTip condition // is evaludated EndAtTipCheckInterval = 10 * time.Second + + //MinTableSize unit is GB + MinTableSize = int64(3) + + //MaxTableSize unit is GB + MaxTableSize = int64(100) + + //MinTableSize unit is MB + MinValueLogFileSize = int64(256) + + //MaxTableSize unit is MB + MaxValueLogFileSize = int64(2048) ) var _ http.Handler = (*DataTester)(nil) @@ -152,17 +164,44 @@ func InitializeData( } opts := []database.BadgerOption{} - if config.CompressionDisabled { - opts = append(opts, database.WithoutCompression()) - } - if config.MemoryLimitDisabled { + dataPathBackup := dataPath + + if config.AllInMemoryEnabled{ opts = append( opts, - database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)), + database.WithoutCompression(), ) + if(config.TableSize != nil) { + if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) { + opts = append( + opts, + database.WithTableSize(*config.TableSize), + ) + } + } + if(config.ValueLogFileSize != nil) { + if(*config.TableSize >= MinValueLogFileSize && *config.TableSize <= MinValueLogFileSize) { + opts = append( + opts, + database.WithValueLogFileSize(*config.TableSize), + ) + } + } + dataPathBackup = "" + } else { + if config.CompressionDisabled { + opts = append(opts, database.WithoutCompression()) + } + if config.MemoryLimitDisabled { + opts = append( + opts, + database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + ) + } } - localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) + localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...) if err != nil { return nil, fmt.Errorf("%s: unable to initialize database", err.Error()) } From 71d29934a8e357d3158bcd4d6d867008878af1d1 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 8 Aug 2022 20:37:05 -0700 Subject: [PATCH 02/10] updates --- examples/configuration/default.json | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index df5c68ab..c2ca45a7 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -15,6 +15,7 @@ "log_configuration": false, "compression_disabled": false, "memory_limit_disabled": false, + "all_in_memory_enabled": false, "error_stack_trace_disabled": false, "coin_supported": false, "construction": null, From 0e53ff336041b1dd68a9175e632a8dba04083cef Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 13:30:01 -0700 Subject: [PATCH 03/10] updates --- cmd/root.go | 26 +++++++++++++++++++++++++- configuration/types.go | 4 ++-- go.mod | 4 +--- go.sum | 4 ++-- pkg/tester/data.go | 29 ++++++++++++++++------------- 5 files changed, 46 insertions(+), 21 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 45984c19..a0fcc0fd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,8 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - + allInMemory bool + tableSize int64 // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set // to the default settings. @@ -263,6 +264,21 @@ default values.`, "", "Data-dir configures the location of logs and data for validation. This will override the data_directory from configuration file", ) + + checkDataCmd.Flags().Int64Var( + &tableSize, + "table_size", + 0, + `Table_size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, + ) + + checkDataCmd.Flags().BoolVar( + &allInMemory, + "all_in_memory", + false, + "All_in_memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", + ) + rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -372,6 +388,14 @@ func initConfig() { if len(dataDirectory) != 0 { Config.DataDirectory = dataDirectory } + + if allInMemory { + Config.AllInMemoryEnabled = allInMemory + } + + if tableSize != 0 { + Config.TableSize = &tableSize + } } func ensureDataDirectoryExists() { diff --git a/configuration/types.go b/configuration/types.go index 3f3af83f..93c8d4db 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -421,12 +421,12 @@ type Configuration struct { // TableSize unit is GB, enable users to define MaxTableSize // when AllInMemoryEnabled == true, Cli will look up this config - // default value is 6, modification range is [3, 100] + // default value is 6, modification range is [2, 100] TableSize *int64 `json:"table_size,omitempty"` // TableSize unit is MB, enable users to define ValueLogFileSize // when AllInMemoryEnabled == true, Cli will look up this config - // default value is 512, modification range is [256, 2048] + // default value is 512, modification range is [128, 2048] ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"` // SeenBlockWorkers is the number of goroutines spawned to store diff --git a/go.mod b/go.mod index ac5cbc7e..dcb81ec5 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,8 @@ module github.com/coinbase/rosetta-cli go 1.16 -replace github.com/coinbase/rosetta-sdk-go => /Users/jingweizhuang/crypto/rosetta-sdk-go - require ( - github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 + github.com/coinbase/rosetta-sdk-go v0.7.11 github.com/fatih/color v1.13.0 github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 // indirect diff --git a/go.sum b/go.sum index 8797de37..4db8ed05 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 h1:cHRCEzEk+4xerPtuJgO5a71qp7jIwGDKCKq+pte2VHA= -github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= +github.com/coinbase/rosetta-sdk-go v0.7.11 h1:T7camDGstlX2ENVE0QHR1AVu3asSXED0vg1xDaF6bYE= +github.com/coinbase/rosetta-sdk-go v0.7.11/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 23662230..42bac274 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -67,13 +67,13 @@ const ( EndAtTipCheckInterval = 10 * time.Second //MinTableSize unit is GB - MinTableSize = int64(3) + MinTableSize = int64(2) //MaxTableSize unit is GB MaxTableSize = int64(100) //MinTableSize unit is MB - MinValueLogFileSize = int64(256) + MinValueLogFileSize = int64(128) //MaxTableSize unit is MB MaxValueLogFileSize = int64(2048) @@ -172,6 +172,20 @@ func InitializeData( database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)), database.WithoutCompression(), ) + dataPathBackup = "" + } else { + if config.CompressionDisabled { + opts = append(opts, database.WithoutCompression()) + } + if config.MemoryLimitDisabled { + opts = append( + opts, + database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + ) + } + } + + if config.AllInMemoryEnabled || config.MemoryLimitDisabled { if(config.TableSize != nil) { if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) { opts = append( @@ -188,17 +202,6 @@ func InitializeData( ) } } - dataPathBackup = "" - } else { - if config.CompressionDisabled { - opts = append(opts, database.WithoutCompression()) - } - if config.MemoryLimitDisabled { - opts = append( - opts, - database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), - ) - } } localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...) From c3a4f4be5f4b91a04bfe39e1de8574d89a57cef7 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 13:41:51 -0700 Subject: [PATCH 04/10] updates --- cmd/root.go | 1 + configuration/types.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a0fcc0fd..ad48879b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -57,6 +57,7 @@ var ( dataDirectory string allInMemory bool tableSize int64 + // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set // to the default settings. diff --git a/configuration/types.go b/configuration/types.go index 93c8d4db..2534cbc3 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -416,16 +416,16 @@ type Configuration struct { // AllInMemoryDisabled configures storage to increase memory // usage. Enabling this massively increases performance - // but can use 20s of GBs of RAM, even with pruning enabled. + // but can use >20s of GBs of RAM, even with pruning enabled. AllInMemoryEnabled bool `json:"all_in_memory_enabled"` // TableSize unit is GB, enable users to define MaxTableSize - // when AllInMemoryEnabled == true, Cli will look up this config + // when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config // default value is 6, modification range is [2, 100] TableSize *int64 `json:"table_size,omitempty"` // TableSize unit is MB, enable users to define ValueLogFileSize - // when AllInMemoryEnabled == true, Cli will look up this config + // when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config // default value is 512, modification range is [128, 2048] ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"` From dccf2c45393327a49a3e7c535547339e0aee3931 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 14:20:51 -0700 Subject: [PATCH 05/10] updates --- pkg/tester/data.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 42bac274..12ed5a9d 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -172,6 +172,7 @@ func InitializeData( database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)), database.WithoutCompression(), ) + // for all in memory mode, the path need to be "", as badgerDB will not write to disk dataPathBackup = "" } else { if config.CompressionDisabled { @@ -185,6 +186,10 @@ func InitializeData( } } + // If we enable all-in-memory or L0-in-memory mode, badger DB's TableSize and ValueLogFileSize will change + // according to users config. tableSize means the LSM table size, when the table more than the tableSize, + // will trigger a compact. + // In default mode, we will not change the badger DB's TableSize and ValueLogFileSize for limiting memory usage if config.AllInMemoryEnabled || config.MemoryLimitDisabled { if(config.TableSize != nil) { if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) { From aa77764295e50252c20b4e0cbca955c830bdc449 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 15:59:51 -0700 Subject: [PATCH 06/10] update --- cmd/root.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index ad48879b..b362879e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -268,16 +268,16 @@ default values.`, checkDataCmd.Flags().Int64Var( &tableSize, - "table_size", + "table-size", 0, - `Table_size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, + `Table-size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, ) checkDataCmd.Flags().BoolVar( &allInMemory, - "all_in_memory", + "all-in-memory", false, - "All_in_memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", + "All-in-memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", ) rootCmd.AddCommand(checkDataCmd) From a6397808eb34229738ae21e51336b6a6191d1a6b Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 22:57:11 -0700 Subject: [PATCH 07/10] update --- cmd/root.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index b362879e..3e349f4c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,7 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - allInMemory bool + turnOnMemoryMode bool tableSize int64 // Config is the populated *configuration.Configuration from @@ -270,14 +270,14 @@ default values.`, &tableSize, "table-size", 0, - `Table-size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, + `Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file`, ) checkDataCmd.Flags().BoolVar( - &allInMemory, - "all-in-memory", + &turnOnMemoryMode, + "turn-on-memory-mode", false, - "All-in-memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", + "turn-on-memory-mode configures badger DB inMeomry option. if turn-on-memory-mode=true This will override the all_in_memory_enabled", ) rootCmd.AddCommand(checkDataCmd) @@ -390,8 +390,8 @@ func initConfig() { Config.DataDirectory = dataDirectory } - if allInMemory { - Config.AllInMemoryEnabled = allInMemory + if turnOnMemoryMode { + Config.AllInMemoryEnabled = turnOnMemoryMode } if tableSize != 0 { From dfa1d2d960f1e044fe1fc4ca572c28e6b02fc0e6 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 15 Aug 2022 12:53:06 -0700 Subject: [PATCH 08/10] updates --- cmd/root.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 3e349f4c..a5065b92 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,7 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - turnOnMemoryMode bool + inOnMemoryMode bool tableSize int64 // Config is the populated *configuration.Configuration from @@ -274,10 +274,10 @@ default values.`, ) checkDataCmd.Flags().BoolVar( - &turnOnMemoryMode, - "turn-on-memory-mode", + &inOnMemoryMode, + "in-memory-mode", false, - "turn-on-memory-mode configures badger DB inMeomry option. if turn-on-memory-mode=true This will override the all_in_memory_enabled", + "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true This will override the all_in_memory_enabled", ) rootCmd.AddCommand(checkDataCmd) @@ -390,8 +390,8 @@ func initConfig() { Config.DataDirectory = dataDirectory } - if turnOnMemoryMode { - Config.AllInMemoryEnabled = turnOnMemoryMode + if inOnMemoryMode { + Config.AllInMemoryEnabled = inOnMemoryMode } if tableSize != 0 { From 0af17a2104eccc63168761e8edb6cbcbbf557a36 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 15 Aug 2022 13:21:28 -0700 Subject: [PATCH 09/10] updates --- cmd/root.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a5065b92..6507a74d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,7 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - inOnMemoryMode bool + inMemoryMode bool tableSize int64 // Config is the populated *configuration.Configuration from @@ -274,7 +274,7 @@ default values.`, ) checkDataCmd.Flags().BoolVar( - &inOnMemoryMode, + &inMemoryMode, "in-memory-mode", false, "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true This will override the all_in_memory_enabled", @@ -390,8 +390,8 @@ func initConfig() { Config.DataDirectory = dataDirectory } - if inOnMemoryMode { - Config.AllInMemoryEnabled = inOnMemoryMode + if inMemoryMode { + Config.AllInMemoryEnabled = inMemoryMode } if tableSize != 0 { From bd732bce2183e7f9bf4cd840be833416cf5164be Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 15 Aug 2022 13:29:45 -0700 Subject: [PATCH 10/10] updates --- cmd/root.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6507a74d..a6b6cb6d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -270,14 +270,14 @@ default values.`, &tableSize, "table-size", 0, - `Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file`, + "Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file", ) checkDataCmd.Flags().BoolVar( &inMemoryMode, "in-memory-mode", false, - "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true This will override the all_in_memory_enabled", + "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled", ) rootCmd.AddCommand(checkDataCmd)