From 6d0643c6505edfdc454a8fc6ef3ff2933ee0a25b Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Fri, 22 Sep 2023 17:47:10 -0400 Subject: [PATCH] cli: add --experimental-secondary-cache flag This change adds a new `--experimental-secondary-cache` flag for use with `--experimental-shared-storage` to enable the use of a secondary cache to speed up reads of objects in shared storage. This option sets the max cache size for each store using disaggregated storage; for per-store granularity, pebble options can also be used. Epic: none Release note: None --- pkg/cli/cliflags/flags.go | 23 +++++++++++++++++++++++ pkg/cli/flags.go | 1 + pkg/server/config.go | 5 +++++ pkg/storage/ballast.go | 13 +++++++++++++ pkg/storage/open.go | 13 ++++++++----- 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/pkg/cli/cliflags/flags.go b/pkg/cli/cliflags/flags.go index d00e3d0c8ffc..9653bed3a44e 100644 --- a/pkg/cli/cliflags/flags.go +++ b/pkg/cli/cliflags/flags.go @@ -1024,6 +1024,29 @@ only present for backward compatibility. `, } + SecondaryCache = FlagInfo{ + Name: "experimental-secondary-cache", + Description: ` +Enables the use of a secondary cache to store objects from shared storage (see +--experimental-shared-storage) inside local paths for each store. A size must +be specified with this flag, which will be the maximum size for the secondary +cache on each store on this node: +
+
+  --experimental-secondary-cache=20GiB
+
+
+The size can be given in various ways: +
+
+  --experimental-secondary-cache=10000000000     -> 10000000000 bytes
+  --experimental-secondary-cache=20GB            -> 20000000000 bytes
+  --experimental-secondary-cache=20GiB           -> 21474836480 bytes
+  --experimental-secondary-cache=20%             -> 20% of available space
+  --experimental-secondary-cache=0.2             -> 20% of available space
+  --experimental-secondary-cache=.2              -> 20% of available space
`, + } + SharedStorage = FlagInfo{ Name: "experimental-shared-storage", Description: fmt.Sprintf(` diff --git a/pkg/cli/flags.go b/pkg/cli/flags.go index 4b7c495bb057..ef5e53056180 100644 --- a/pkg/cli/flags.go +++ b/pkg/cli/flags.go @@ -489,6 +489,7 @@ func init() { cliflagcfg.VarFlag(f, &storeSpecs, cliflags.Store) cliflagcfg.VarFlag(f, &serverCfg.StorageEngine, cliflags.StorageEngine) cliflagcfg.StringFlag(f, &serverCfg.SharedStorage, cliflags.SharedStorage) + cliflagcfg.VarFlag(f, &serverCfg.SecondaryCache, cliflags.SecondaryCache) cliflagcfg.VarFlag(f, &serverCfg.MaxOffset, cliflags.MaxOffset) cliflagcfg.BoolFlag(f, &serverCfg.DisableMaxOffsetCheck, cliflags.DisableMaxOffsetCheck) cliflagcfg.StringFlag(f, &serverCfg.ClockDevicePath, cliflags.ClockDevice) diff --git a/pkg/server/config.go b/pkg/server/config.go index 966b21d7c254..15ab6f4e08a5 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -231,6 +231,10 @@ type BaseConfig struct { SharedStorage string *cloud.ExternalStorageAccessor + // SecondaryCache is the size of the secondary cache used for each store, to + // store blocks from disaggregated shared storage. For use with SharedStorage. + SecondaryCache base.SizeSpec + // StartDiagnosticsReporting starts the asynchronous goroutine that // checks for CockroachDB upgrades and periodically reports // diagnostics to Cockroach Labs. @@ -818,6 +822,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { if sharedStorage != nil { addCfgOpt(storage.SharedStorage(sharedStorage)) } + addCfgOpt(storage.SecondaryCache(storage.SecondaryCacheBytes(cfg.SecondaryCache, du))) // If the spec contains Pebble options, set those too. if spec.PebbleOptions != "" { addCfgOpt(storage.PebbleOptions(spec.PebbleOptions, &pebble.ParseHooks{ diff --git a/pkg/storage/ballast.go b/pkg/storage/ballast.go index c4bda616e0ec..8cf4a5ebe421 100644 --- a/pkg/storage/ballast.go +++ b/pkg/storage/ballast.go @@ -103,6 +103,19 @@ func BallastSizeBytes(spec base.StoreSpec, diskUsage vfs.DiskUsage) int64 { return v } +// SecondaryCacheBytes returns the desired size of the secondary cache, calculated +// from the provided store spec and disk usage. If the store spec contains an +// explicit ballast size (either in bytes or as a percentage of the disk's total +// capacity), that size is used. A zero value for cacheSize results in no +// secondary cache. +func SecondaryCacheBytes(cacheSize base.SizeSpec, diskUsage vfs.DiskUsage) int64 { + v := cacheSize.InBytes + if cacheSize.Percent != 0 { + v = int64(float64(diskUsage.TotalBytes) * cacheSize.Percent / 100) + } + return v +} + func maybeEstablishBallast( fs vfs.FS, ballastPath string, ballastSizeBytes int64, diskUsage vfs.DiskUsage, ) (resized bool, err error) { diff --git a/pkg/storage/open.go b/pkg/storage/open.go index c71c590281b9..68d858256e30 100644 --- a/pkg/storage/open.go +++ b/pkg/storage/open.go @@ -171,11 +171,14 @@ func BallastSize(size int64) ConfigOption { func SharedStorage(sharedStorage cloud.ExternalStorage) ConfigOption { return func(cfg *engineConfig) error { cfg.SharedStorage = sharedStorage - // TODO(bilal): Do the format major version ratchet while accounting for - // version upgrade finalization. However, seeing as shared storage is - // an experimental feature and upgrading from existing stores is not - // supported, this is fine. - cfg.Opts.FormatMajorVersion = pebble.FormatVirtualSSTables + return nil + } +} + +// SecondaryCache enables use of a secondary cache to store shared objects. +func SecondaryCache(size int64) ConfigOption { + return func(cfg *engineConfig) error { + cfg.Opts.Experimental.SecondaryCacheSizeBytes = size return nil } }