cli: add --experimental-secondary-cache flag

This change adds a new `--experimental-secondary-cache`
flag for use with `--experimental-shared-storage` to enable
the use of a secondary cache to speed up reads of objects
in shared storage. This option sets the max cache size for
each store using disaggregated storage; for per-store
granularity, pebble options can also be used.

Epic: none

Release note: None

pkg/cli/cliflags/flags.go

@@ -1024,6 +1024,30 @@ only present for backward compatibility.
 `,
 	}
 
+	SecondaryCache = FlagInfo{
+		Name: "experimental-secondary-cache",
+		Description: `
+Enables the use of a secondary cache to store objects from shared storage (see
+--experimental-shared-storage) inside local paths for each store. A size must
+be specified with this flag, which will be the maximum size for the secondary
+cache on each store on this node:
+<PRE>
+
+  --experimental-secondary-cache=20GiB
+
+</PRE>
+The size can be given in various ways:
+<PRE>
+
+  --experimental-secondary-cache=10000000000     -> 10000000000 bytes
+  --experimental-secondary-cache=20GB            -> 20000000000 bytes
+  --experimental-secondary-cache=20GiB           -> 21474836480 bytes
+  --experimental-secondary-cache=0.02TiB         -> 21474836480 bytes
+  --experimental-secondary-cache=20%             -> 20% of available space
+  --experimental-secondary-cache=0.2             -> 20% of available space
+  --experimental-secondary-cache=.2              -> 20% of available space</PRE>`,
+	}
+
 	SharedStorage = FlagInfo{
 		Name: "experimental-shared-storage",
 		Description: fmt.Sprintf(`

pkg/cli/flags.go

@@ -489,6 +489,7 @@ func init() {
 			cliflagcfg.VarFlag(f, &storeSpecs, cliflags.Store)
 			cliflagcfg.VarFlag(f, &serverCfg.StorageEngine, cliflags.StorageEngine)
 			cliflagcfg.StringFlag(f, &serverCfg.SharedStorage, cliflags.SharedStorage)
+			cliflagcfg.VarFlag(f, &serverCfg.SecondaryCache, cliflags.SecondaryCache)
 			cliflagcfg.VarFlag(f, &serverCfg.MaxOffset, cliflags.MaxOffset)
 			cliflagcfg.BoolFlag(f, &serverCfg.DisableMaxOffsetCheck, cliflags.DisableMaxOffsetCheck)
 			cliflagcfg.StringFlag(f, &serverCfg.ClockDevicePath, cliflags.ClockDevice)

pkg/server/config.go

@@ -236,6 +236,10 @@ type BaseConfig struct {
 	SharedStorage string
 	*cloud.ExternalStorageAccessor
 
+	// SecondaryCache is the size of the secondary cache used for each store, to
+	// store blocks from disaggregated shared storage. For use with SharedStorage.
+	SecondaryCache base.SizeSpec
+
 	// StartDiagnosticsReporting starts the asynchronous goroutine that
 	// checks for CockroachDB upgrades and periodically reports
 	// diagnostics to Cockroach Labs.
@@ -826,6 +830,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) {
 			if sharedStorage != nil {
 				addCfgOpt(storage.SharedStorage(sharedStorage))
 			}
+			addCfgOpt(storage.SecondaryCache(storage.SecondaryCacheBytes(cfg.SecondaryCache, du)))
 			// If the spec contains Pebble options, set those too.
 			if spec.PebbleOptions != "" {
 				addCfgOpt(storage.PebbleOptions(spec.PebbleOptions, &pebble.ParseHooks{

pkg/storage/ballast.go

@@ -103,6 +103,19 @@ func BallastSizeBytes(spec base.StoreSpec, diskUsage vfs.DiskUsage) int64 {
 	return v
 }
 
+// SecondaryCacheBytes returns the desired size of the secondary cache, calculated
+// from the provided store spec and disk usage. If the store spec contains an
+// explicit ballast size (either in bytes or as a percentage of the disk's total
+// capacity), that size is used. A zero value for cacheSize results in no
+// secondary cache.
+func SecondaryCacheBytes(cacheSize base.SizeSpec, diskUsage vfs.DiskUsage) int64 {
+	v := cacheSize.InBytes
+	if cacheSize.Percent != 0 {
+		v = int64(float64(diskUsage.TotalBytes) * cacheSize.Percent / 100)
+	}
+	return v
+}
+
 func maybeEstablishBallast(
 	fs vfs.FS, ballastPath string, ballastSizeBytes int64, diskUsage vfs.DiskUsage,
 ) (resized bool, err error) {

pkg/storage/open.go

@@ -171,11 +171,14 @@ func BallastSize(size int64) ConfigOption {
 func SharedStorage(sharedStorage cloud.ExternalStorage) ConfigOption {
 	return func(cfg *engineConfig) error {
 		cfg.SharedStorage = sharedStorage
-		// TODO(bilal): Do the format major version ratchet while accounting for
-		// version upgrade finalization. However, seeing as shared storage is
-		// an experimental feature and upgrading from existing stores is not
-		// supported, this is fine.
-		cfg.Opts.FormatMajorVersion = pebble.FormatVirtualSSTables
+		return nil
+	}
+}
+
+// SecondaryCache enables use of a secondary cache to store shared objects.
+func SecondaryCache(size int64) ConfigOption {
+	return func(cfg *engineConfig) error {
+		cfg.Opts.Experimental.SecondaryCacheSizeBytes = size
 		return nil
 	}
 }