diff --git a/internal/entrypoint/run.go b/internal/entrypoint/run.go index c53017b..574360d 100644 --- a/internal/entrypoint/run.go +++ b/internal/entrypoint/run.go @@ -137,7 +137,7 @@ func Run(ctx context.Context, config *Config, exporter otlexporters.Otlexporter, logger.Info("powerscale quota capacity metrics collection is disabled") continue } - powerScaleSvc.ExportVolumeMetrics(ctx) + powerScaleSvc.ExportQuotaMetrics(ctx) case err := <-errCh: if err == nil { continue diff --git a/internal/entrypoint/run_test.go b/internal/entrypoint/run_test.go index 16d3902..e0fca2d 100644 --- a/internal/entrypoint/run_test.go +++ b/internal/entrypoint/run_test.go @@ -45,7 +45,7 @@ func Test_Run(t *testing.T) { e.EXPECT().StopExporter().Return(nil) svc := mocks.NewMockService(ctrl) - svc.EXPECT().ExportVolumeMetrics(gomock.Any()).AnyTimes() + svc.EXPECT().ExportQuotaMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterCapacityMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterPerformanceMetrics(gomock.Any()).AnyTimes() @@ -131,7 +131,7 @@ func Test_Run(t *testing.T) { e.EXPECT().StopExporter().Return(nil) svc := mocks.NewMockService(ctrl) - svc.EXPECT().ExportVolumeMetrics(gomock.Any()).AnyTimes() + svc.EXPECT().ExportQuotaMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterCapacityMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterPerformanceMetrics(gomock.Any()).AnyTimes() @@ -160,7 +160,7 @@ func Test_Run(t *testing.T) { e.EXPECT().StopExporter().Return(nil) svc := mocks.NewMockService(ctrl) - svc.EXPECT().ExportVolumeMetrics(gomock.Any()).AnyTimes() + svc.EXPECT().ExportQuotaMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterCapacityMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterPerformanceMetrics(gomock.Any()).AnyTimes() return false, config, e, svc, prevConfigValidationFunc, ctrl, true @@ -236,7 +236,7 @@ func Test_Run(t *testing.T) { e.EXPECT().StopExporter().Return(nil) svc := mocks.NewMockService(ctrl) - svc.EXPECT().ExportVolumeMetrics(gomock.Any()).AnyTimes() + svc.EXPECT().ExportQuotaMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterCapacityMetrics(gomock.Any()).AnyTimes() svc.EXPECT().ExportClusterPerformanceMetrics(gomock.Any()).AnyTimes() diff --git a/internal/service/metrics.go b/internal/service/metrics.go index d6c1501..086ba4b 100644 --- a/internal/service/metrics.go +++ b/internal/service/metrics.go @@ -21,7 +21,8 @@ import ( // MetricsRecorder supports recording volume and cluster metric //go:generate mockgen -destination=mocks/metrics_mocks.go -package=mocks github.com/dell/csm-metrics-powerscale/internal/service MetricsRecorder,AsyncMetricCreator type MetricsRecorder interface { - RecordVolumeSpace(ctx context.Context, meta interface{}, subscribedQuota, hardQuota int64) error + RecordVolumeQuota(ctx context.Context, meta interface{}, metric *VolumeQuotaMetricsRecord) error + RecordClusterQuota(ctx context.Context, meta interface{}, metric *ClusterQuotaRecord) error RecordClusterCapacityStatsMetrics(ctx context.Context, metric *ClusterCapacityStatsMetricsRecord) error RecordClusterPerformanceStatsMetrics(ctx context.Context, metric *ClusterPerformanceStatsMetricsRecord) error } @@ -41,12 +42,22 @@ type MetricsWrapper struct { VolumeMetrics sync.Map ClusterCapacityStatsMetrics sync.Map ClusterPerformanceStatsMetrics sync.Map + VolumeQuotaMetrics sync.Map + ClusterQuotaMetrics sync.Map } -// VolumeSpaceMetrics contains the volume metrics data -type VolumeSpaceMetrics struct { - QuotaSubscribed asyncfloat64.UpDownCounter - HardQuotaRemaining asyncfloat64.UpDownCounter +// VolumeQuotaMetrics contains volume quota metrics data +type VolumeQuotaMetrics struct { + QuotaSubscribed asyncfloat64.UpDownCounter + HardQuotaRemaining asyncfloat64.UpDownCounter + QuotaSubscribedPct asyncfloat64.UpDownCounter + HardQuotaRemainingPct asyncfloat64.UpDownCounter +} + +// ClusterQuotaMetrics contains quota capacity in all directories +type ClusterQuotaMetrics struct { + TotalHardQuotaGigabytes asyncfloat64.UpDownCounter + TotalHardQuotaPct asyncfloat64.UpDownCounter } // ClusterCapacityStatsMetrics contains the capacity stats metrics related to a cluster @@ -116,7 +127,65 @@ func updateLabels(prefix, metaID string, labels []attribute.KeyValue, mw *Metric return metricsMapValue, nil } -func (mw *MetricsWrapper) initVolumeMetrics(prefix, metaID string, labels []attribute.KeyValue) (*VolumeSpaceMetrics, error) { +func (mw *MetricsWrapper) initClusterQuotaMetrics(prefix, metaID string, labels []attribute.KeyValue) (*ClusterQuotaMetrics, error) { + totalHardQuota, err := mw.Meter.AsyncFloat64().UpDownCounter(prefix + "total_hard_quota_gigabytes") + if err != nil { + return nil, err + } + TotalHardQuotaPct, err := mw.Meter.AsyncFloat64().UpDownCounter(prefix + "total_hard_quota_percentage") + if err != nil { + return nil, err + } + + metrics := &ClusterQuotaMetrics{ + TotalHardQuotaGigabytes: totalHardQuota, + TotalHardQuotaPct: TotalHardQuotaPct, + } + + mw.ClusterQuotaMetrics.Store(metaID, metrics) + mw.Labels.Store(metaID, labels) + + return metrics, nil +} + +// RecordClusterQuota will publish cluster Quota metrics data +func (mw *MetricsWrapper) RecordClusterQuota(ctx context.Context, meta interface{}, metric *ClusterQuotaRecord) error { + var prefix string + var metaID string + var labels []attribute.KeyValue + switch v := meta.(type) { + case *ClusterMeta: + prefix, metaID = "powerscale_directory_", v.ClusterName + labels = []attribute.KeyValue{ + attribute.String("ClusterName", v.ClusterName), + attribute.String("PlotWithMean", "No"), + } + default: + return errors.New("unknown MetaData type") + } + + loadMetricsFunc := func(metaID string) (any, bool) { + return mw.ClusterQuotaMetrics.Load(metaID) + } + + initMetricsFunc := func(prefix string, metaID string, labels []attribute.KeyValue) (any, error) { + return mw.initClusterQuotaMetrics(prefix, metaID, labels) + } + + metricsMapValue, err := updateLabels(prefix, metaID, labels, mw, loadMetricsFunc, initMetricsFunc) + + if err != nil { + return err + } + + metrics := metricsMapValue.(*ClusterQuotaMetrics) + metrics.TotalHardQuotaGigabytes.Observe(ctx, utils.UnitsConvert(metric.totalHardQuota, utils.BYTES, utils.GB), labels...) + metrics.TotalHardQuotaPct.Observe(ctx, metric.totalHardQuotaPct, labels...) + + return nil +} + +func (mw *MetricsWrapper) initVolumeQuotaMetrics(prefix, metaID string, labels []attribute.KeyValue) (*VolumeQuotaMetrics, error) { quotaSubscribed, err := mw.Meter.AsyncFloat64().UpDownCounter(prefix + "quota_subscribed_gigabytes") if err != nil { return nil, err @@ -125,20 +194,30 @@ func (mw *MetricsWrapper) initVolumeMetrics(prefix, metaID string, labels []attr if err != nil { return nil, err } + quotaSubscribedPct, err := mw.Meter.AsyncFloat64().UpDownCounter(prefix + "quota_subscribed_percentage") + if err != nil { + return nil, err + } + hardQuotaRemainingPct, err := mw.Meter.AsyncFloat64().UpDownCounter(prefix + "hard_quota_remaining_percentage") + if err != nil { + return nil, err + } - metrics := &VolumeSpaceMetrics{ - QuotaSubscribed: quotaSubscribed, - HardQuotaRemaining: hardQuotaRemaining, + metrics := &VolumeQuotaMetrics{ + QuotaSubscribed: quotaSubscribed, + HardQuotaRemaining: hardQuotaRemaining, + QuotaSubscribedPct: quotaSubscribedPct, + HardQuotaRemainingPct: hardQuotaRemainingPct, } - mw.VolumeMetrics.Store(metaID, metrics) + mw.VolumeQuotaMetrics.Store(metaID, metrics) mw.Labels.Store(metaID, labels) return metrics, nil } -// RecordVolumeSpace will publish volume metrics data -func (mw *MetricsWrapper) RecordVolumeSpace(ctx context.Context, meta interface{}, subscribedQuota, hardQuotaRemaining int64) error { +// RecordVolumeQuota will publish volume Quota metrics data +func (mw *MetricsWrapper) RecordVolumeQuota(ctx context.Context, meta interface{}, metric *VolumeQuotaMetricsRecord) error { var prefix string var metaID string var labels []attribute.KeyValue @@ -153,18 +232,18 @@ func (mw *MetricsWrapper) RecordVolumeSpace(ctx context.Context, meta interface{ attribute.String("StorageClass", v.StorageClass), attribute.String("PlotWithMean", "No"), attribute.String("PersistentVolumeClaim", v.PersistentVolumeClaimName), - attribute.String("PersistentVolumeNameSpace", v.NameSpace), + attribute.String("Namespace", v.Namespace), } default: return errors.New("unknown MetaData type") } loadMetricsFunc := func(metaID string) (any, bool) { - return mw.VolumeMetrics.Load(metaID) + return mw.VolumeQuotaMetrics.Load(metaID) } initMetricsFunc := func(prefix string, metaID string, labels []attribute.KeyValue) (any, error) { - return mw.initVolumeMetrics(prefix, metaID, labels) + return mw.initVolumeQuotaMetrics(prefix, metaID, labels) } metricsMapValue, err := updateLabels(prefix, metaID, labels, mw, loadMetricsFunc, initMetricsFunc) @@ -173,9 +252,11 @@ func (mw *MetricsWrapper) RecordVolumeSpace(ctx context.Context, meta interface{ return err } - metrics := metricsMapValue.(*VolumeSpaceMetrics) - metrics.QuotaSubscribed.Observe(ctx, utils.UnitsConvert(subscribedQuota, utils.BYTES, utils.GB), labels...) - metrics.HardQuotaRemaining.Observe(ctx, utils.UnitsConvert(hardQuotaRemaining, utils.BYTES, utils.GB), labels...) + metrics := metricsMapValue.(*VolumeQuotaMetrics) + metrics.QuotaSubscribed.Observe(ctx, utils.UnitsConvert(metric.quotaSubscribed, utils.BYTES, utils.GB), labels...) + metrics.HardQuotaRemaining.Observe(ctx, utils.UnitsConvert(metric.hardQuotaRemaining, utils.BYTES, utils.GB), labels...) + metrics.QuotaSubscribedPct.Observe(ctx, metric.quotaSubscribedPct, labels...) + metrics.HardQuotaRemainingPct.Observe(ctx, metric.hardQuotaRemainingPct, labels...) return nil } diff --git a/internal/service/metrics_test.go b/internal/service/metrics_test.go index fca60d5..78e488f 100644 --- a/internal/service/metrics_test.go +++ b/internal/service/metrics_test.go @@ -22,7 +22,7 @@ import ( "go.opentelemetry.io/otel/metric/global" ) -func Test_Metrics_Record(t *testing.T) { +func Test_Volume_Quota_Metrics_Record(t *testing.T) { type checkFn func(*testing.T, error) checkFns := func(checkFns ...checkFn) []checkFn { return checkFns } @@ -54,13 +54,17 @@ func Test_Metrics_Record(t *testing.T) { otMeter := global.Meter(prefix + "_test") subscribed, err := otMeter.AsyncFloat64().UpDownCounter(prefix + "subscribed_quota") hardQuota, err := otMeter.AsyncFloat64().UpDownCounter(prefix + "hard_quota_remaining_gigabytes") + subscribedPct, err := otMeter.AsyncFloat64().UpDownCounter(prefix + "subscribed_quota_percentage") + hardQuotaPct, err := otMeter.AsyncFloat64().UpDownCounter(prefix + "hard_quota_remaining_percentage") if err != nil { t.Fatal(err) } - meter.EXPECT().AsyncFloat64().Return(provider).Times(2) + meter.EXPECT().AsyncFloat64().Return(provider).Times(4) provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(subscribed, nil) provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(hardQuota, nil) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(subscribedPct, nil) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(hardQuotaPct, nil) return &service.MetricsWrapper{ Meter: meter, @@ -77,7 +81,7 @@ func Test_Metrics_Record(t *testing.T) { ctrl := gomock.NewController(t) getMeter := func(prefix string) *service.MetricsWrapper { meter := mocks.NewMockAsyncMetricCreator(ctrl) - provider := mocks.NewMockInstrumentProvider(ctrl) + provider := asyncfloat64mock.NewMockInstrumentProvider(ctrl) otMeter := global.Meter(prefix + "_test") subscribed, err := otMeter.AsyncFloat64().UpDownCounter(prefix + "quota_subscribed_gigabytes") if err != nil { @@ -131,7 +135,127 @@ func Test_Metrics_Record(t *testing.T) { t.Run(name, func(t *testing.T) { mws, checks := tc(t) for i := range mws { - err := mws[i].RecordVolumeSpace(context.Background(), metas[i], 10000, 2000) + err := mws[i].RecordVolumeQuota(context.Background(), metas[i], &service.VolumeQuotaMetricsRecord{}) + for _, check := range checks { + check(t, err) + } + } + }) + } +} + +func Test_Cluster_Quota_Metrics_Record(t *testing.T) { + type checkFn func(*testing.T, error) + checkFns := func(checkFns ...checkFn) []checkFn { return checkFns } + + verifyError := func(t *testing.T, err error) { + if err == nil { + t.Errorf("expected an error, got nil") + } + } + + verifyNoError := func(t *testing.T, err error) { + if err != nil { + t.Errorf("expected nil error, got %v", err) + } + } + + metas := []interface{}{ + &service.ClusterMeta{ + ClusterName: "cluster-1", + }, + } + + tests := map[string]func(t *testing.T) ([]*service.MetricsWrapper, []checkFn){ + "success": func(t *testing.T) ([]*service.MetricsWrapper, []checkFn) { + ctrl := gomock.NewController(t) + + getMeter := func(prefix string) *service.MetricsWrapper { + meter := mocks.NewMockAsyncMetricCreator(ctrl) + provider := asyncfloat64mock.NewMockInstrumentProvider(ctrl) + otMeter := global.Meter(prefix + "_test") + + clusterSubscribed, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_directory_total_hard_quota_gigabytes") + clusterSubscribedPct, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_directory_total_hard_quota_percentage") + if err != nil { + t.Fatal(err) + } + + meter.EXPECT().AsyncFloat64().Return(provider).Times(2) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(clusterSubscribed, nil) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(clusterSubscribedPct, nil) + + return &service.MetricsWrapper{ + Meter: meter, + } + } + + mws := []*service.MetricsWrapper{ + getMeter("powerscale_directory"), + } + + return mws, checkFns(verifyNoError) + }, + "error creating cluster_subscribed_quota": func(t *testing.T) ([]*service.MetricsWrapper, []checkFn) { + ctrl := gomock.NewController(t) + getMeter := func(prefix string) *service.MetricsWrapper { + meter := mocks.NewMockAsyncMetricCreator(ctrl) + provider := asyncfloat64mock.NewMockInstrumentProvider(ctrl) + otMeter := global.Meter(prefix + "_test") + clusterSubscribed, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_directory_total_hard_quota_gigabytes") + + if err != nil { + t.Fatal(err) + } + + meter.EXPECT().AsyncFloat64().Return(provider).Times(1) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(clusterSubscribed, errors.New("error")) + + return &service.MetricsWrapper{ + Meter: meter, + } + } + + mws := []*service.MetricsWrapper{ + getMeter("powerscale_directory_"), + } + + return mws, checkFns(verifyError) + }, + "error creating hard_quota_remaining": func(t *testing.T) ([]*service.MetricsWrapper, []checkFn) { + ctrl := gomock.NewController(t) + getMeter := func(prefix string) *service.MetricsWrapper { + meter := mocks.NewMockAsyncMetricCreator(ctrl) + provider := asyncfloat64mock.NewMockInstrumentProvider(ctrl) + otMeter := global.Meter(prefix + "_test") + clusterSubscribed, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_directory_total_hard_quota_gigabytes") + clusterSubscribedPct, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_directory_total_hard_quota_percentage") + if err != nil { + t.Fatal(err) + } + + meter.EXPECT().AsyncFloat64().Return(provider).Times(2) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(clusterSubscribed, nil) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(clusterSubscribedPct, errors.New("error")) + + return &service.MetricsWrapper{ + Meter: meter, + } + } + + mws := []*service.MetricsWrapper{ + getMeter("powerscale_directory_"), + } + + return mws, checkFns(verifyError) + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + mws, checks := tc(t) + for i := range mws { + err := mws[i].RecordClusterQuota(context.Background(), metas[i], &service.ClusterQuotaRecord{}) for _, check := range checks { check(t, err) } @@ -148,7 +272,7 @@ func Test_Volume_Metrics_Label_Update(t *testing.T) { IsiPath: "/ifs/data/csi", StorageClass: "isilon", PersistentVolumeClaimName: "pvc-name", - NameSpace: "pvc-namespace", + Namespace: "pvc-namespace", } metaSecond := &service.VolumeMeta{ @@ -158,7 +282,7 @@ func Test_Volume_Metrics_Label_Update(t *testing.T) { IsiPath: "/ifs/data/csi", StorageClass: "isilon", PersistentVolumeClaimName: "pvc-name", - NameSpace: "pvc-namespace", + Namespace: "pvc-namespace", } metaThird := &service.VolumeMeta{ @@ -168,7 +292,7 @@ func Test_Volume_Metrics_Label_Update(t *testing.T) { IsiPath: "/ifs/data/csi", StorageClass: "isilon", PersistentVolumeClaimName: "pvc-name", - NameSpace: "pvc-namespace", + Namespace: "pvc-namespace", } expectedLabels := []attribute.KeyValue{ @@ -179,38 +303,42 @@ func Test_Volume_Metrics_Label_Update(t *testing.T) { attribute.String("StorageClass", metaSecond.StorageClass), attribute.String("PlotWithMean", "No"), attribute.String("PersistentVolumeClaim", metaSecond.PersistentVolumeClaimName), - attribute.String("PersistentVolumeNameSpace", metaSecond.NameSpace), + attribute.String("PersistentVolumeNameSpace", metaSecond.Namespace), } ctrl := gomock.NewController(t) meter := mocks.NewMockAsyncMetricCreator(ctrl) provider := asyncfloat64mock.NewMockInstrumentProvider(ctrl) - otMeter := global.Meter("powerscale_volume_test") + otMeter := global.Meter("powerscale_volume_quota_test") subscribed, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_volume_quota_subscribed_gigabytes") hardQuota, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_volume_hard_quota_remaining_gigabytes") + subscribedPct, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_volume_quota_subscribed_quota_percentage") + hardQuotaPct, err := otMeter.AsyncFloat64().UpDownCounter("powerscale_volume_hard_quota_remaining_percentage") if err != nil { t.Fatal(err) } - meter.EXPECT().AsyncFloat64().Return(provider).Times(4) + meter.EXPECT().AsyncFloat64().Return(provider).Times(8) provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(subscribed, nil).Times(2) provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(hardQuota, nil).Times(2) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(subscribedPct, nil).Times(2) + provider.EXPECT().UpDownCounter(gomock.Any(), gomock.Any()).Return(hardQuotaPct, nil).Times(2) mw := &service.MetricsWrapper{ Meter: meter, } - t.Run("success: volume metric labels updated", func(t *testing.T) { - err := mw.RecordVolumeSpace(context.Background(), metaFirst, 1000, 2000) + t.Run("success: metric labels updated", func(t *testing.T) { + err := mw.RecordVolumeQuota(context.Background(), metaFirst, &service.VolumeQuotaMetricsRecord{}) if err != nil { t.Errorf("expected nil error (record #1), got %v", err) } - err = mw.RecordVolumeSpace(context.Background(), metaSecond, 1000, 2000) + err = mw.RecordVolumeQuota(context.Background(), metaSecond, &service.VolumeQuotaMetricsRecord{}) if err != nil { t.Errorf("expected nil error (record #2), got %v", err) } - err = mw.RecordVolumeSpace(context.Background(), metaThird, 1000, 2000) + err = mw.RecordVolumeQuota(context.Background(), metaThird, &service.VolumeQuotaMetricsRecord{}) if err != nil { t.Errorf("expected nil error (record #3), got %v", err) } diff --git a/internal/service/mocks/metrics_mocks.go b/internal/service/mocks/metrics_mocks.go index 337d641..ec82371 100644 --- a/internal/service/mocks/metrics_mocks.go +++ b/internal/service/mocks/metrics_mocks.go @@ -65,18 +65,32 @@ func (mr *MockMetricsRecorderMockRecorder) RecordClusterPerformanceStatsMetrics( return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordClusterPerformanceStatsMetrics", reflect.TypeOf((*MockMetricsRecorder)(nil).RecordClusterPerformanceStatsMetrics), arg0, arg1) } -// RecordVolumeSpace mocks base method. -func (m *MockMetricsRecorder) RecordVolumeSpace(arg0 context.Context, arg1 interface{}, arg2, arg3 int64) error { +// RecordClusterQuota mocks base method. +func (m *MockMetricsRecorder) RecordClusterQuota(arg0 context.Context, arg1 interface{}, arg2 *service.ClusterQuotaRecord) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecordVolumeSpace", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "RecordClusterQuota", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } -// RecordVolumeSpace indicates an expected call of RecordVolumeSpace. -func (mr *MockMetricsRecorderMockRecorder) RecordVolumeSpace(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +// RecordClusterQuota indicates an expected call of RecordClusterQuota. +func (mr *MockMetricsRecorderMockRecorder) RecordClusterQuota(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordVolumeSpace", reflect.TypeOf((*MockMetricsRecorder)(nil).RecordVolumeSpace), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordClusterQuota", reflect.TypeOf((*MockMetricsRecorder)(nil).RecordClusterQuota), arg0, arg1, arg2) +} + +// RecordVolumeQuota mocks base method. +func (m *MockMetricsRecorder) RecordVolumeQuota(arg0 context.Context, arg1 interface{}, arg2 *service.VolumeQuotaMetricsRecord) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecordVolumeQuota", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecordVolumeQuota indicates an expected call of RecordVolumeQuota. +func (mr *MockMetricsRecorderMockRecorder) RecordVolumeQuota(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordVolumeQuota", reflect.TypeOf((*MockMetricsRecorder)(nil).RecordVolumeQuota), arg0, arg1, arg2) } // MockAsyncMetricCreator is a mock of AsyncMetricCreator interface. diff --git a/internal/service/mocks/powerscale_client_mocks.go b/internal/service/mocks/powerscale_client_mocks.go index 77dc9c1..04bce5b 100644 --- a/internal/service/mocks/powerscale_client_mocks.go +++ b/internal/service/mocks/powerscale_client_mocks.go @@ -64,18 +64,3 @@ func (mr *MockPowerScaleClientMockRecorder) GetFloatStatistics(arg0, arg1 interf mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFloatStatistics", reflect.TypeOf((*MockPowerScaleClient)(nil).GetFloatStatistics), arg0, arg1) } - -// GetQuotaWithPath mocks base method. -func (m *MockPowerScaleClient) GetQuotaWithPath(arg0 context.Context, arg1 string) (goisilon.Quota, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetQuotaWithPath", arg0, arg1) - ret0, _ := ret[0].(goisilon.Quota) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetQuotaWithPath indicates an expected call of GetQuotaWithPath. -func (mr *MockPowerScaleClientMockRecorder) GetQuotaWithPath(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetQuotaWithPath", reflect.TypeOf((*MockPowerScaleClient)(nil).GetQuotaWithPath), arg0, arg1) -} diff --git a/internal/service/mocks/service_mocks.go b/internal/service/mocks/service_mocks.go index 5bc0bf5..d53636b 100644 --- a/internal/service/mocks/service_mocks.go +++ b/internal/service/mocks/service_mocks.go @@ -58,14 +58,14 @@ func (mr *MockServiceMockRecorder) ExportClusterPerformanceMetrics(arg0 interfac return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportClusterPerformanceMetrics", reflect.TypeOf((*MockService)(nil).ExportClusterPerformanceMetrics), arg0) } -// ExportVolumeMetrics mocks base method. -func (m *MockService) ExportVolumeMetrics(arg0 context.Context) { +// ExportQuotaMetrics mocks base method. +func (m *MockService) ExportQuotaMetrics(arg0 context.Context) { m.ctrl.T.Helper() - m.ctrl.Call(m, "ExportVolumeMetrics", arg0) + m.ctrl.Call(m, "ExportQuotaMetrics", arg0) } -// ExportVolumeMetrics indicates an expected call of ExportVolumeMetrics. -func (mr *MockServiceMockRecorder) ExportVolumeMetrics(arg0 interface{}) *gomock.Call { +// ExportQuotaMetrics indicates an expected call of ExportQuotaMetrics. +func (mr *MockServiceMockRecorder) ExportQuotaMetrics(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportVolumeMetrics", reflect.TypeOf((*MockService)(nil).ExportVolumeMetrics), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportQuotaMetrics", reflect.TypeOf((*MockService)(nil).ExportQuotaMetrics), arg0) } diff --git a/internal/service/service.go b/internal/service/service.go index 47de4b7..0d703b4 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -28,12 +28,14 @@ const ( DefaultMaxPowerScaleConnections = 10 // ExpectedVolumeHandleProperties is the number of properties that the VolumeHandle contains ExpectedVolumeHandleProperties = 4 + // DirectoryQuotaType is the type of Quota corresponding to a volume + DirectoryQuotaType = "directory" ) // Service contains operations that would be used to interact with a PowerScale system //go:generate mockgen -destination=mocks/service_mocks.go -package=mocks github.com/dell/csm-metrics-powerscale/internal/service Service type Service interface { - ExportVolumeMetrics(context.Context) + ExportQuotaMetrics(context.Context) ExportClusterCapacityMetrics(context.Context) ExportClusterPerformanceMetrics(context.Context) } @@ -43,9 +45,6 @@ type Service interface { type PowerScaleClient interface { GetFloatStatistics(ctx context.Context, keys []string) (goisilon.FloatStats, error) GetAllQuotas(ctx context.Context) (goisilon.QuotaList, error) - - // GetQuotaWithPath returns Quota data by path - GetQuotaWithPath(ctx context.Context, path string) (goisilon.Quota, error) } // PowerScaleService represents the service for getting metrics data for a PowerScale system @@ -99,20 +98,29 @@ type ClusterPerformanceStatsMetricsRecord struct { DirectoryTotalHardQuotaPercentage float64 } -// VolumeMetricsRecord used for holding output of the Volume stat query results -type VolumeMetricsRecord struct { - volumeMeta *VolumeMeta - quotaSubscribed int64 - hardQuotaRemaining int64 +// VolumeQuotaMetricsRecord used for holding output of the Volume stat query results +type VolumeQuotaMetricsRecord struct { + volumeMeta *VolumeMeta + quotaSubscribed int64 + hardQuotaRemaining int64 + quotaSubscribedPct float64 + hardQuotaRemainingPct float64 +} + +// ClusterQuotaRecord used for holding output of the Volume stat query results +type ClusterQuotaRecord struct { + clusterMeta *ClusterMeta + totalHardQuota int64 + totalHardQuotaPct float64 } -// ExportVolumeMetrics records space metrics for the given list of Volumes -func (s *PowerScaleService) ExportVolumeMetrics(ctx context.Context) { +// ExportQuotaMetrics records quota metrics for the given list of Volumes +func (s *PowerScaleService) ExportQuotaMetrics(ctx context.Context) { start := time.Now() - defer s.timeSince(start, "ExportVolumeMetrics") + defer s.timeSince(start, "ExportQuotaMetrics") if s.MetricsWrapper == nil { - s.Logger.Warn("no MetricsWrapper provided for getting ExportVolumeMetrics") + s.Logger.Warn("no MetricsWrapper provided for getting ExportQuotaMetrics") return } @@ -127,9 +135,143 @@ func (s *PowerScaleService) ExportVolumeMetrics(ctx context.Context) { return } - for range s.pushVolumeMetrics(ctx, s.gatherVolumeMetrics(ctx, s.volumeServer(ctx, pvs))) { - // consume the channel until it is empty and closed + cluster2Quotas := make(map[string]goisilon.QuotaList) + for clusterName, client := range s.PowerScaleClients { + quotaList, err := client.GetAllQuotas(ctx) + if err != nil { + s.Logger.WithError(err).WithField("cluster_name", clusterName).Error("getting quotas") + continue + } + cluster2Quotas[clusterName] = quotaList } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + for range s.pushVolumeQuotaMetrics(ctx, s.gatherVolumeQuotaMetrics(ctx, cluster2Quotas, s.volumeServer(ctx, pvs))) { + // consume the channel until it is empty and closed + } + wg.Done() + }() + + go func() { + for range s.pushClusterQuotaMetrics(ctx, s.gatherClusterQuotaMetrics(ctx, cluster2Quotas)) { + // consume the channel until it is empty and closed + } + wg.Done() + }() + + wg.Wait() +} + +// pushClusterQuotaMetrics will push the provided channel of cluster quota metrics to a data collector +func (s *PowerScaleService) pushClusterQuotaMetrics(ctx context.Context, clusterQuotaMetrics <-chan *ClusterQuotaRecord) <-chan string { + start := time.Now() + defer s.timeSince(start, "pushClusterQuotaMetrics") + var wg sync.WaitGroup + + ch := make(chan string) + go func() { + for metrics := range clusterQuotaMetrics { + wg.Add(1) + go func(metrics *ClusterQuotaRecord) { + defer wg.Done() + err := s.MetricsWrapper.RecordClusterQuota(ctx, metrics.clusterMeta, metrics) + if err != nil { + s.Logger.WithError(err).WithField("cluster_name", metrics.clusterMeta.ClusterName).Error("recording quota metrics for cluster") + } else { + ch <- fmt.Sprintf(metrics.clusterMeta.ClusterName) + } + }(metrics) + } + wg.Wait() + close(ch) + }() + + return ch +} + +// gatherClusterQuotaMetrics will return a channel of volume metrics based on the input of volumes +func (s *PowerScaleService) gatherClusterQuotaMetrics(ctx context.Context, cluster2Quotas map[string]goisilon.QuotaList) <-chan *ClusterQuotaRecord { + start := time.Now() + defer s.timeSince(start, "gatherClusterQuotaMetrics") + + ch := make(chan *ClusterQuotaRecord) + var wg sync.WaitGroup + sem := make(chan struct{}, s.MaxPowerScaleConnections) + + go func() { + for clusterName := range s.PowerScaleClients { + sem <- struct{}{} + wg.Add(1) + meta := ClusterMeta{ClusterName: clusterName} + go func(meta ClusterMeta) { + defer func() { + wg.Done() + <-sem + }() + quotaList := cluster2Quotas[meta.ClusterName] + if len(quotaList) == 0 { + return + } + highestLevelQuotas := getHighestQuotas(quotaList) + totalHardQuota := int64(0) + totalHardQuotaUsage := int64(0) + for _, quota := range highestLevelQuotas { + totalHardQuota = totalHardQuota + quota.Thresholds.Hard + totalHardQuotaUsage = totalHardQuotaUsage + quota.Usage.Logical + } + + totalHardQuotaPct := float64(0) + if totalHardQuota != 0 { + totalHardQuotaPct = float64(totalHardQuotaUsage) * 100.0 / float64(totalHardQuota) + } + + metric := &ClusterQuotaRecord{ + clusterMeta: &meta, + totalHardQuota: totalHardQuota, + totalHardQuotaPct: totalHardQuotaPct, + } + s.Logger.Debugf("cluster quota metrics %+v", *metric) + + ch <- metric + }(meta) + } + wg.Wait() + close(ch) + close(sem) + }() + return ch +} + +func getHighestQuotas(list goisilon.QuotaList) goisilon.QuotaList { + highestQuotas := make(goisilon.QuotaList, 0) + for _, quota := range list { + if quota.Type != DirectoryQuotaType { + continue + } + isSubLevel := false + + for hIndex := 0; hIndex < len(highestQuotas); hIndex++ { + // current Quota's level is higher,remove current highest quota + if strings.Contains(highestQuotas[hIndex].Path, quota.Path+"/") { + if hIndex == len(highestQuotas)-1 { + highestQuotas = highestQuotas[:hIndex] + } else { + highestQuotas = append(highestQuotas[:hIndex], highestQuotas[hIndex+1:]...) + } + hIndex-- + } else if strings.Contains(quota.Path, highestQuotas[hIndex].Path+"/") { + // current Quota is a children of known Quota + isSubLevel = true + break + } + } + if !isSubLevel { + highestQuotas = append(highestQuotas, quota) + } + } + return highestQuotas } // volumeServer will return a channel of volumes that can provide statistics about each volume @@ -144,12 +286,13 @@ func (s *PowerScaleService) volumeServer(ctx context.Context, volumes []k8s.Volu return volumeChannel } -// gatherVolumeMetrics will return a channel of volume metrics based on the input of volumes -func (s *PowerScaleService) gatherVolumeMetrics(ctx context.Context, volumes <-chan k8s.VolumeInfo) <-chan *VolumeMetricsRecord { +// gatherVolumeQuotaMetrics will return a channel of volume metrics based on the input of volumes +func (s *PowerScaleService) gatherVolumeQuotaMetrics(ctx context.Context, cluster2Quotas map[string]goisilon.QuotaList, + volumes <-chan k8s.VolumeInfo) <-chan *VolumeQuotaMetricsRecord { start := time.Now() - defer s.timeSince(start, "gatherVolumeMetrics") + defer s.timeSince(start, "gatherVolumeQuotaMetrics") - ch := make(chan *VolumeMetricsRecord) + ch := make(chan *VolumeQuotaMetricsRecord) var wg sync.WaitGroup sem := make(chan struct{}, s.MaxPowerScaleConnections) @@ -195,7 +338,7 @@ func (s *PowerScaleService) gatherVolumeMetrics(ctx context.Context, volumes <-c Driver: volume.Driver, IsiPath: volume.IsiPath, PersistentVolumeClaimName: volume.VolumeClaimName, - NameSpace: volume.Namespace, + Namespace: volume.Namespace, } if volumeMeta.IsiPath == "" { @@ -210,15 +353,15 @@ func (s *PowerScaleService) gatherVolumeMetrics(ctx context.Context, volumes <-c } } - goPowerScaleClient, err := s.getPowerScaleClient(ctx, clusterName) - if err != nil { - s.Logger.WithError(err).WithField("cluster_name", clusterName).Warn("no client found for PowerScale with clsuter_name") - return - } - path := volumeMeta.IsiPath + "/" + volumeID - volQuota, err := goPowerScaleClient.GetQuotaWithPath(ctx, path) - if err != nil { + var volQuota goisilon.Quota + for _, q := range cluster2Quotas[clusterName] { + if q.Path == path && q.Type == DirectoryQuotaType { + volQuota = q + break + } + } + if volQuota == nil { s.Logger.WithError(err).WithField("volume_id", volumeMeta.ID).Error("getting quota metrics") return } @@ -226,17 +369,23 @@ func (s *PowerScaleService) gatherVolumeMetrics(ctx context.Context, volumes <-c subscribedQuota := volQuota.Usage.Logical hardQuotaRemaining := volQuota.Thresholds.Hard - volQuota.Usage.Logical - s.Logger.WithFields(logrus.Fields{ - "volume_meta": volumeMeta, - "quota_subscribed_gigabytes": subscribedQuota, - "quota_hard_quato_remaining_gigabytes": hardQuotaRemaining, - }).Debug("volume quota metrics") + subscribedQuotaPct := float64(0) + hardQuotaRemainingPct := float64(0) + if volQuota.Thresholds.Hard != 0 { + subscribedQuotaPct = float64(subscribedQuota) * 100.0 / float64(volQuota.Thresholds.Hard) + hardQuotaRemainingPct = float64(hardQuotaRemaining) * 100.0 / float64(volQuota.Thresholds.Hard) + } - ch <- &VolumeMetricsRecord{ - volumeMeta: volumeMeta, - quotaSubscribed: subscribedQuota, - hardQuotaRemaining: hardQuotaRemaining, + metric := &VolumeQuotaMetricsRecord{ + volumeMeta: volumeMeta, + quotaSubscribed: subscribedQuota, + hardQuotaRemaining: hardQuotaRemaining, + quotaSubscribedPct: subscribedQuotaPct, + hardQuotaRemainingPct: hardQuotaRemainingPct, } + s.Logger.Debugf("volume quota metrics %+v", *metric) + + ch <- metric }(volume) } @@ -247,23 +396,19 @@ func (s *PowerScaleService) gatherVolumeMetrics(ctx context.Context, volumes <-c return ch } -// pushVolumeMetrics will push the provided channel of volume metrics to a data collector -func (s *PowerScaleService) pushVolumeMetrics(ctx context.Context, volumeMetrics <-chan *VolumeMetricsRecord) <-chan string { +// pushVolumeQuotaMetrics will push the provided channel of volume metrics to a data collector +func (s *PowerScaleService) pushVolumeQuotaMetrics(ctx context.Context, volumeMetrics <-chan *VolumeQuotaMetricsRecord) <-chan string { start := time.Now() - defer s.timeSince(start, "pushVolumeMetrics") + defer s.timeSince(start, "pushVolumeQuotaMetrics") var wg sync.WaitGroup ch := make(chan string) go func() { for metrics := range volumeMetrics { wg.Add(1) - go func(metrics *VolumeMetricsRecord) { + go func(metrics *VolumeQuotaMetricsRecord) { defer wg.Done() - err := s.MetricsWrapper.RecordVolumeSpace(ctx, - metrics.volumeMeta, - metrics.quotaSubscribed, - metrics.hardQuotaRemaining, - ) + err := s.MetricsWrapper.RecordVolumeQuota(ctx, metrics.volumeMeta, metrics) if err != nil { s.Logger.WithError(err).WithField("volume_id", metrics.volumeMeta.ID).Error("recording metrics for volume") } else { diff --git a/internal/service/service_test.go b/internal/service/service_test.go index c70691a..8a0225c 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -13,86 +13,78 @@ import ( "errors" "github.com/dell/goisilon" "github.com/dell/goisilon/api/json" - isiV1 "github.com/dell/goisilon/api/v1" "io/ioutil" + v1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "testing" "github.com/dell/csm-metrics-powerscale/internal/service" "github.com/dell/csm-metrics-powerscale/internal/service/mocks" "github.com/sirupsen/logrus" - v1 "k8s.io/api/storage/v1" "github.com/dell/csm-metrics-powerscale/internal/k8s" "github.com/golang/mock/gomock" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +var mockVolumes = []k8s.VolumeInfo{ + { + Namespace: "karavi", + PersistentVolumeClaim: "pvc-uid", + PersistentVolumeStatus: "Bound", + VolumeClaimName: "pvc-name", + PersistentVolume: "pv-1", + StorageClass: "isilon", + Driver: "csi-isilon.dellemc.com", + ProvisionedSize: "16Gi", + VolumeHandle: "mock-pv1=_=_=19=_=_=System=_=_=cluster1", + IsiPath: "/ifs/data/csi", + }, + { + Namespace: "karavi", + PersistentVolumeClaim: "pvc-uid", + PersistentVolumeStatus: "Bound", + VolumeClaimName: "pvc-name", + PersistentVolume: "pv-2", + StorageClass: "isilon-another", + Driver: "csi-isilon.dellemc.com", + ProvisionedSize: "16Gi", + VolumeHandle: "mock-pv2=_=_=19=_=_=System=_=_=cluster1", + IsiPath: "/ifs/data/csi", + }, + { + Namespace: "karavi", + PersistentVolumeClaim: "pvc-uid", + PersistentVolumeStatus: "Bound", + VolumeClaimName: "pvc-name", + PersistentVolume: "pv-3", + StorageClass: "isilon", + Driver: "csi-isilon.dellemc.com", + ProvisionedSize: "16Gi", + VolumeHandle: "mock-pv3=_=_=19=_=_=System=_=_=cluster2", + IsiPath: "/ifs/data/csi", + }, +} + func Test_ExportVolumeMetrics(t *testing.T) { - // Mock data for volume space - mockQuota := &isiV1.IsiQuota{ - Usage: struct { - Inodes int64 `json:"inodes"` - Logical int64 `json:"logical"` - Physical int64 `json:"physical"` - }{Logical: 1000, Physical: 2000}, - Thresholds: struct { - Advisory int64 `json:"advisory"` - AdvisoryExceeded bool `json:"advisory_exceeded"` - AdvisoryLastExceeded interface{} `json:"advisory_last_exceeded"` - Hard int64 `json:"hard"` - HardExceeded bool `json:"hard_exceeded"` - HardLastExceeded interface{} `json:"hard_last_exceeded"` - Soft int64 `json:"soft"` - SoftExceeded bool `json:"soft_exceeded"` - SoftLastExceeded interface{} `json:"soft_last_exceeded"` - }{Hard: 2000}, - } + quotaFile1 := "testdata/recordings/client1-quotas.json" + contentBytes1, _ := ioutil.ReadFile(quotaFile1) + var client1MockQuotaList goisilon.QuotaList + json.Unmarshal(contentBytes1, &client1MockQuotaList) + + quotaFile2 := "testdata/recordings/client2-quotas.json" + contentBytes2, _ := ioutil.ReadFile(quotaFile2) + var client2MockQuotaList goisilon.QuotaList + json.Unmarshal(contentBytes2, &client2MockQuotaList) tests := map[string]func(t *testing.T) (service.PowerScaleService, *gomock.Controller){ "success": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(3) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(3) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(2) volFinder := mocks.NewMockVolumeFinder(ctrl) - volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return([]k8s.VolumeInfo{ - { - Namespace: "karavi", - PersistentVolumeClaim: "pvc-uid", - PersistentVolumeStatus: "Bound", - VolumeClaimName: "pvc-name", - PersistentVolume: "pv-1", - StorageClass: "isilon", - Driver: "csi-isilon.dellemc.com", - ProvisionedSize: "16Gi", - VolumeHandle: "k8s-7242537ae1=_=_=19=_=_=System=_=_=cluster1", - IsiPath: "/ifs/data/csi", - }, - { - Namespace: "karavi", - PersistentVolumeClaim: "pvc-uid", - PersistentVolumeStatus: "Bound", - VolumeClaimName: "pvc-name", - PersistentVolume: "pv-2", - StorageClass: "isilon-another", - Driver: "csi-isilon.dellemc.com", - ProvisionedSize: "16Gi", - VolumeHandle: "k8s-7242537ae1=_=_=19=_=_=System=_=_=cluster2", - IsiPath: "/ifs/data/csi", - }, - { - Namespace: "karavi", - PersistentVolumeClaim: "pvc-uid", - PersistentVolumeStatus: "Bound", - VolumeClaimName: "pvc-name", - PersistentVolume: "pv-3", - StorageClass: "isilon", - Driver: "csi-isilon.dellemc.com", - ProvisionedSize: "16Gi", - VolumeHandle: "k8s-7242537ae1=_=_=19=_=_=System=_=_=cluster1", - IsiPath: "/ifs/data/csi", - }, - }, nil).Times(1) + volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return(mockVolumes, nil).Times(1) scFinder := mocks.NewMockStorageClassFinder(ctrl) scFinder.EXPECT().GetStorageClasses(gomock.Any()).Return([]v1.StorageClass{ @@ -124,9 +116,9 @@ func Test_ExportVolumeMetrics(t *testing.T) { clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Return(mockQuota, nil).Times(2) + client1.EXPECT().GetAllQuotas(gomock.Any()).Return(client1MockQuotaList, nil).Times(1) client2 := mocks.NewMockPowerScaleClient(ctrl) - client2.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Return(mockQuota, nil).Times(1) + client2.EXPECT().GetAllQuotas(gomock.Any()).Return(client2MockQuotaList, nil).Times(1) clients["cluster1"] = client1 clients["cluster2"] = client2 @@ -141,7 +133,8 @@ func Test_ExportVolumeMetrics(t *testing.T) { "success but volume isiPath is defaultIsiPath": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) volFinder := mocks.NewMockVolumeFinder(ctrl) volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return([]k8s.VolumeInfo{ @@ -154,7 +147,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { StorageClass: "isilon", Driver: "csi-isilon.dellemc.com", ProvisionedSize: "16Gi", - VolumeHandle: "k8s-7242537ae1=_=_=19=_=_=System=_=_=cluster1", + VolumeHandle: "mock-pv1=_=_=19=_=_=System=_=_=cluster1", }, }, nil).Times(1) @@ -174,7 +167,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { }, nil).Times(1) clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Return(mockQuota, nil).Times(1) + client1.EXPECT().GetAllQuotas(gomock.Any()).Return(client1MockQuotaList, nil).Times(1) clients["cluster1"] = client1 service := service.PowerScaleService{ @@ -183,27 +176,18 @@ func Test_ExportVolumeMetrics(t *testing.T) { StorageClassFinder: scFinder, PowerScaleClients: clients, } + service.ClientIsiPaths = make(map[string]string) + service.ClientIsiPaths["cluster1"] = "/ifs/data/csi" return service, ctrl }, - "metrics not pushed if error getting quota": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { + "quota metrics not pushed if error getting quota": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) volFinder := mocks.NewMockVolumeFinder(ctrl) - volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return([]k8s.VolumeInfo{ - { - Namespace: "karavi", - PersistentVolumeClaim: "pvc-uid", - PersistentVolumeStatus: "Bound", - VolumeClaimName: "pvc-name", - PersistentVolume: "pv-1", - StorageClass: "isilon", - Driver: "csi-isilon.dellemc.com", - ProvisionedSize: "16Gi", - VolumeHandle: "k8s-7242537ae1=_=_=19=_=_=System=_=_=cluster1", - }, - }, nil).Times(1) + volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return(mockVolumes[:0], nil).Times(1) scFinder := mocks.NewMockStorageClassFinder(ctrl) scFinder.EXPECT().GetStorageClasses(gomock.Any()).Return([]v1.StorageClass{ @@ -222,7 +206,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Return(nil, errors.New("error")).Times(1) + client1.EXPECT().GetAllQuotas(gomock.Any()).Return(nil, errors.New("error")).Times(1) clients["cluster1"] = client1 service := service.PowerScaleService{ @@ -234,25 +218,14 @@ func Test_ExportVolumeMetrics(t *testing.T) { return service, ctrl }, - "metrics not pushed if no cluster name in volume handle": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { + "quota metrics not pushed if no cluster name in volume handle": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) volFinder := mocks.NewMockVolumeFinder(ctrl) - volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return([]k8s.VolumeInfo{ - { - Namespace: "karavi", - PersistentVolumeClaim: "pvc-uid", - PersistentVolumeStatus: "Bound", - VolumeClaimName: "pvc-name", - PersistentVolume: "pv-1", - StorageClass: "isilon", - Driver: "csi-isilon.dellemc.com", - ProvisionedSize: "16Gi", - VolumeHandle: "k8s-7242537ae1=_=_=19=_=_=System=_=_=cluster1", - }, - }, nil).Times(1) + volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return(mockVolumes[:0], nil).Times(1) scFinder := mocks.NewMockStorageClassFinder(ctrl) scFinder.EXPECT().GetStorageClasses(gomock.Any()).Return([]v1.StorageClass{ @@ -271,7 +244,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Times(0) + client1.EXPECT().GetAllQuotas(gomock.Any()).Times(1) clients["cluster2"] = client1 service := service.PowerScaleService{ @@ -285,7 +258,8 @@ func Test_ExportVolumeMetrics(t *testing.T) { "metrics not pushed if volume handle is invalid": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) volFinder := mocks.NewMockVolumeFinder(ctrl) volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return([]k8s.VolumeInfo{ @@ -319,7 +293,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Times(0) + client1.EXPECT().GetAllQuotas(gomock.Any()).Times(1) clients["cluster1"] = client1 service := service.PowerScaleService{ @@ -330,10 +304,11 @@ func Test_ExportVolumeMetrics(t *testing.T) { } return service, ctrl }, - "metrics not pushed if volume finder returns error": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { + "quota metrics not pushed if volume finder returns error": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) volFinder := mocks.NewMockVolumeFinder(ctrl) volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return(nil, errors.New("error")).Times(1) @@ -342,7 +317,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Times(0) + client1.EXPECT().GetAllQuotas(gomock.Any()).Times(0) clients["cluster1"] = client1 service := service.PowerScaleService{ @@ -353,32 +328,20 @@ func Test_ExportVolumeMetrics(t *testing.T) { } return service, ctrl }, - "metrics not pushed if storage class finder returns error": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { + "volume quota metrics not pushed if storage class finder returns error": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) volFinder := mocks.NewMockVolumeFinder(ctrl) - volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return([]k8s.VolumeInfo{ - { - Namespace: "karavi", - PersistentVolumeClaim: "pvc-uid", - PersistentVolumeStatus: "Bound", - VolumeClaimName: "pvc-name", - PersistentVolume: "pv-1", - StorageClass: "isilon", - Driver: "csi-isilon.dellemc.com", - ProvisionedSize: "16Gi", - VolumeHandle: "k8s-7242537ae1=_=_=19=_=_=System=_=_=cluster1", - }, - }, nil).Times(1) - + volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Return(mockVolumes[:0], nil).Times(1) scFinder := mocks.NewMockStorageClassFinder(ctrl) scFinder.EXPECT().GetStorageClasses(gomock.Any()).Return(nil, errors.New("error")).Times(1) clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Return(mockQuota, nil).Times(1) + client1.EXPECT().GetAllQuotas(gomock.Any()).Return(client1MockQuotaList, nil).Times(1) clients["cluster1"] = client1 service := service.PowerScaleService{ @@ -389,7 +352,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { } return service, ctrl }, - "metrics not pushed if metrics wrapper is nil": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { + "quota metrics not pushed if metrics wrapper is nil": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) volFinder := mocks.NewMockVolumeFinder(ctrl) volFinder.EXPECT().GetPersistentVolumes(gomock.Any()).Times(0) @@ -398,7 +361,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetQuotaWithPath(gomock.Any(), gomock.Any()).Times(0) + client1.EXPECT().GetAllQuotas(gomock.Any()).Times(0) clients["cluster1"] = client1 service := service.PowerScaleService{ @@ -409,10 +372,11 @@ func Test_ExportVolumeMetrics(t *testing.T) { } return service, ctrl }, - "metrics not pushed with 0 volumes": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { + "quota metrics not pushed with 0 volumes": func(*testing.T) (service.PowerScaleService, *gomock.Controller) { ctrl := gomock.NewController(t) metrics := mocks.NewMockMetricsRecorder(ctrl) - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordClusterQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) scFinder := mocks.NewMockStorageClassFinder(ctrl) scFinder.EXPECT().GetStorageClasses(gomock.Any()).Return([]v1.StorageClass{ { @@ -434,7 +398,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { clients := make(map[string]service.PowerScaleClient) client1 := mocks.NewMockPowerScaleClient(ctrl) - client1.EXPECT().GetFloatStatistics(gomock.Any(), gomock.Any()).Times(0) + client1.EXPECT().GetAllQuotas(gomock.Any()).Times(1) clients["cluster1"] = client1 service := service.PowerScaleService{ @@ -443,7 +407,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { StorageClassFinder: scFinder, PowerScaleClients: clients, } - metrics.EXPECT().RecordVolumeSpace(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + metrics.EXPECT().RecordVolumeQuota(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) return service, ctrl }, } @@ -451,7 +415,7 @@ func Test_ExportVolumeMetrics(t *testing.T) { t.Run(name, func(t *testing.T) { service, ctrl := tc(t) service.Logger = logrus.New() - service.ExportVolumeMetrics(context.Background()) + service.ExportQuotaMetrics(context.Background()) ctrl.Finish() }) } diff --git a/internal/service/testdata/recordings/client1-quotas.json b/internal/service/testdata/recordings/client1-quotas.json new file mode 100644 index 0000000..c48d088 --- /dev/null +++ b/internal/service/testdata/recordings/client1-quotas.json @@ -0,0 +1,194 @@ +[ + { + "container": true, + "enforced": true, + "id": "pwHJAAEAAAAAAAAAAAAAQE0BAAAAAAAA", + "include_snapshots": false, + "linked": false, + "notifications": "default", + "path": "/ifs/data/csi/mock-pv1", + "persona": null, + "ready": true, + "thresholds": { + "advisory": null, + "advisory_exceeded": false, + "advisory_last_exceeded": null, + "hard": 3221225472, + "hard_exceeded": false, + "hard_last_exceeded": null, + "percent_advisory": null, + "percent_soft": null, + "soft": null, + "soft_exceeded": false, + "soft_grace": null, + "soft_last_exceeded": null + }, + "thresholds_include_overhead": false, + "type": "user", + "usage": { + "inodes": 1, + "logical": 0, + "physical": 2048 + } + }, + { + "container": true, + "enforced": true, + "id": "pwHJAAEAAAAAAAAAAAAAQE0BAAAAAAAA", + "include_snapshots": false, + "linked": false, + "notifications": "default", + "path": "/ifs/data/csi/mock-pv1", + "persona": null, + "ready": true, + "thresholds": { + "advisory": null, + "advisory_exceeded": false, + "advisory_last_exceeded": null, + "hard": 3221225472, + "hard_exceeded": false, + "hard_last_exceeded": null, + "percent_advisory": null, + "percent_soft": null, + "soft": null, + "soft_exceeded": false, + "soft_grace": null, + "soft_last_exceeded": null + }, + "thresholds_include_overhead": false, + "type": "directory", + "usage": { + "inodes": 1, + "logical": 0, + "physical": 2048 + } + }, + { + "container": true, + "enforced": true, + "id": "qAHJAAEAAAAAAAAAAAAAQE4BAAAAAAAA", + "include_snapshots": false, + "linked": false, + "notifications": "default", + "path": "/ifs/data/csi/mock-pv2/sub1", + "persona": null, + "ready": true, + "thresholds": { + "advisory": null, + "advisory_exceeded": false, + "advisory_last_exceeded": null, + "hard": 3221225472, + "hard_exceeded": false, + "hard_last_exceeded": null, + "percent_advisory": null, + "percent_soft": null, + "soft": null, + "soft_exceeded": false, + "soft_grace": null, + "soft_last_exceeded": null + }, + "thresholds_include_overhead": false, + "type": "directory", + "usage": { + "inodes": 1, + "logical": 0, + "physical": 2048 + } + }, + { + "container": true, + "enforced": true, + "id": "qgHJAAEAAAAAAAAAAAAAQFABAAAAAAAA", + "include_snapshots": false, + "linked": false, + "notifications": "default", + "path": "/ifs/data/csi/mock-pv2/sub2", + "persona": null, + "ready": true, + "thresholds": { + "advisory": null, + "advisory_exceeded": false, + "advisory_last_exceeded": null, + "hard": 3221225472, + "hard_exceeded": false, + "hard_last_exceeded": null, + "percent_advisory": null, + "percent_soft": null, + "soft": null, + "soft_exceeded": false, + "soft_grace": null, + "soft_last_exceeded": null + }, + "thresholds_include_overhead": false, + "type": "directory", + "usage": { + "inodes": 1, + "logical": 0, + "physical": 2048 + } + }, + { + "container": true, + "enforced": true, + "id": "qQHJAAEAAAAAAAAAAAAAQE8BAAAAAAAA", + "include_snapshots": false, + "linked": false, + "notifications": "default", + "path": "/ifs/data/csi/mock-pv2", + "persona": null, + "ready": true, + "thresholds": { + "advisory": null, + "advisory_exceeded": false, + "advisory_last_exceeded": null, + "hard": 32212254722, + "hard_exceeded": false, + "hard_last_exceeded": null, + "percent_advisory": null, + "percent_soft": null, + "soft": null, + "soft_exceeded": false, + "soft_grace": null, + "soft_last_exceeded": null + }, + "thresholds_include_overhead": false, + "type": "directory", + "usage": { + "inodes": 1, + "logical": 0, + "physical": 2048 + } + }, + { + "container": true, + "enforced": true, + "id": "qgHJAAEAAAAAAAAAAAAAQFABAAAAAAAA", + "include_snapshots": false, + "linked": false, + "notifications": "default", + "path": "/ifs/data/csi/mock-pv2/sub3", + "persona": null, + "ready": true, + "thresholds": { + "advisory": null, + "advisory_exceeded": false, + "advisory_last_exceeded": null, + "hard": 3221225472, + "hard_exceeded": false, + "hard_last_exceeded": null, + "percent_advisory": null, + "percent_soft": null, + "soft": null, + "soft_exceeded": false, + "soft_grace": null, + "soft_last_exceeded": null + }, + "thresholds_include_overhead": false, + "type": "directory", + "usage": { + "inodes": 1, + "logical": 0, + "physical": 2048 + } + } +] \ No newline at end of file diff --git a/internal/service/testdata/recordings/client2-quotas.json b/internal/service/testdata/recordings/client2-quotas.json new file mode 100644 index 0000000..053a10d --- /dev/null +++ b/internal/service/testdata/recordings/client2-quotas.json @@ -0,0 +1,34 @@ +[ + { + "container": true, + "enforced": true, + "id": "pwHJAAEAAAAAAAAAAAAAQE0BAAAAAAAA", + "include_snapshots": false, + "linked": false, + "notifications": "default", + "path": "/ifs/data/csi/mock-pv3", + "persona": null, + "ready": true, + "thresholds": { + "advisory": null, + "advisory_exceeded": false, + "advisory_last_exceeded": null, + "hard": 3221225472, + "hard_exceeded": false, + "hard_last_exceeded": null, + "percent_advisory": null, + "percent_soft": null, + "soft": null, + "soft_exceeded": false, + "soft_grace": null, + "soft_last_exceeded": null + }, + "thresholds_include_overhead": false, + "type": "directory", + "usage": { + "inodes": 1, + "logical": 0, + "physical": 2048 + } + } +] \ No newline at end of file diff --git a/internal/service/types.go b/internal/service/types.go index 35ba730..3a4c9d2 100644 --- a/internal/service/types.go +++ b/internal/service/types.go @@ -12,7 +12,7 @@ import ( "github.com/dell/goisilon" ) -// VolumeMeta is the details of a volume in an SDC +// VolumeMeta is the details of a volume type VolumeMeta struct { ID string PersistentVolumeName string @@ -23,7 +23,12 @@ type VolumeMeta struct { Driver string IsiPath string PersistentVolumeClaimName string - NameSpace string + Namespace string +} + +// ClusterMeta is the details of a cluster +type ClusterMeta struct { + ClusterName string } // PowerScaleCluster is a struct that stores all PowerScale connection information.