bulk: Add metrics to bulk mem monitor for better visualisation

This change reworks how we want to use memory monitors for bulk
operations.

We have a single `bulkMonitor` which is a child of the
root sql memory monitor. Each bulk operation will further create a
child of the `bulkMonitor` which will be used to control memory
growth of the bulk adder.
A new class of metrics has been hooked up to the `bulkMonitor`
which allows for visualization of total memory usage in the
admin UI.

Release note: None

pkg/server/server.go

@@ -370,6 +370,15 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
 	)
 	rootSQLMemoryMonitor.Start(context.Background(), nil, mon.MakeStandaloneBudget(s.cfg.SQLMemoryPoolSize))
 
+	// bulkMemoryMonitor is the parent to all child SQL monitors tracking bulk
+	// operations (IMPORT, index backfill). It is itself a child of the
+	// ParentMemoryMonitor.
+	bulkMemoryMonitor := mon.MakeMonitorInheritWithLimit("bulk-mon", 0 /* limit */, &rootSQLMemoryMonitor)
+	bulkMetrics := bulk.MakeBulkMetrics(cfg.HistogramWindowInterval())
+	s.registry.AddMetricStruct(bulkMetrics)
+	bulkMemoryMonitor.SetMetrics(bulkMetrics.CurBytesCount, bulkMetrics.MaxBytesHist)
+	bulkMemoryMonitor.Start(context.Background(), &rootSQLMemoryMonitor, mon.BoundAccount{})
+
 	// Set up the DistSQL temp engine.
 
 	useStoreSpec := cfg.Stores.Specs[s.cfg.TempStorageConfig.SpecIdx]
@@ -537,17 +546,17 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
 		ClusterName:    s.cfg.ClusterName,
 
 		TempStorage: tempEngine,
+		DiskMonitor: s.cfg.TempStorageConfig.Mon,
+
+		ParentMemoryMonitor: &rootSQLMemoryMonitor,
 		BulkAdder: func(
 			ctx context.Context, db *client.DB, ts hlc.Timestamp, opts storagebase.BulkAdderOptions,
 		) (storagebase.BulkAdder, error) {
 			// Attach a child memory monitor to enable control over the BulkAdder's
 			// memory usage.
-			bulkMon := distsqlrun.NewMonitor(ctx, &rootSQLMemoryMonitor, fmt.Sprintf("bulk-monitor"))
+			bulkMon := distsqlrun.NewMonitor(ctx, &bulkMemoryMonitor, fmt.Sprintf("bulk-adder-monitor"))
 			return bulk.MakeBulkAdder(ctx, db, s.distSender.RangeDescriptorCache(), ts, opts, bulkMon)
 		},
-		DiskMonitor: s.cfg.TempStorageConfig.Mon,
-
-		ParentMemoryMonitor: &rootSQLMemoryMonitor,
 
 		Metrics: &distSQLMetrics,
 

pkg/storage/bulk/bulk_metrics.go

@@ -0,0 +1,56 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package bulk
+
+import (
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/util/metric"
+)
+
+// Metrics contains pointers to the metrics for
+// monitoring bulk operations.
+type Metrics struct {
+	MaxBytesHist  *metric.Histogram
+	CurBytesCount *metric.Gauge
+}
+
+// MetricStruct implements the metrics.Struct interface.
+func (Metrics) MetricStruct() {}
+
+var _ metric.Struct = Metrics{}
+
+var (
+	metaMemMaxBytes = metric.Metadata{
+		Name:        "sql.mem.bulk.max",
+		Help:        "Memory usage per sql statement for bulk operations",
+		Measurement: "Memory",
+		Unit:        metric.Unit_BYTES,
+	}
+	metaMemCurBytes = metric.Metadata{
+		Name:        "sql.mem.bulk.current",
+		Help:        "Current sql statement memory usage for bulk operations",
+		Measurement: "Memory",
+		Unit:        metric.Unit_BYTES,
+	}
+)
+
+// See pkg/sql/mem_metrics.go
+// log10int64times1000 = log10(math.MaxInt64) * 1000, rounded up somewhat
+const log10int64times1000 = 19 * 1000
+
+// MakeBulkMetrics instantiates the metrics holder for bulk operation monitoring.
+func MakeBulkMetrics(histogramWindow time.Duration) Metrics {
+	return Metrics{
+		MaxBytesHist:  metric.NewHistogram(metaMemMaxBytes, histogramWindow, log10int64times1000, 3),
+		CurBytesCount: metric.NewGauge(metaMemCurBytes),
+	}
+}

pkg/ts/catalog/chart_catalog.go

@@ -1374,6 +1374,19 @@ var charts = []sectionDescription{
 			},
 		},
 	},
+	{
+		Organization: [][]string{{SQLLayer, "Bulk"}},
+		Charts: []chartDescription{
+			{
+				Title:   "Current Memory Usage",
+				Metrics: []string{"sql.mem.bulk.current"},
+			},
+			{
+				Title:   "Memory Usage per Statement",
+				Metrics: []string{"sql.mem.bulk.max"},
+			},
+		},
+	},
 	{
 		Organization: [][]string{{SQLLayer, "Optimizer"}},
 		Charts: []chartDescription{

pkg/util/mon/bytes_usage.go

@@ -428,6 +428,12 @@ func (mm *BytesMonitor) AllocBytes() int64 {
 	return mm.mu.curAllocated
 }
 
+// SetMetrics sets the metric objects for the monitor.
+func (mm *BytesMonitor) SetMetrics(curCount *metric.Gauge, maxHist *metric.Histogram) {
+	mm.curBytesCount = curCount
+	mm.maxBytesHist = maxHist
+}
+
 // BoundAccount tracks the cumulated allocations for one client of a pool or
 // monitor. BytesMonitor has an account to its pool; BytesMonitor clients have
 // an account to the monitor. This allows each client to release all the bytes