From a9d9611daf7ee174fd457f6efa692bc4fa8ff9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Fri, 22 Nov 2019 11:05:50 -0800 Subject: [PATCH 1/4] Report volume usage metrics I based this work on https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/commit/ead31529bc35ea360aaa8996277a7eeb77410069 implementation. --- pkg/driver/node.go | 84 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 2dc3dc28f..7310c20d9 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -19,9 +19,11 @@ package driver import ( "context" "fmt" + "golang.org/x/sys/unix" "os" "path/filepath" "regexp" + "strconv" "strings" csi "github.com/container-storage-interface/spec/lib/go/csi" @@ -344,7 +346,64 @@ func (d *nodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } func (d *nodeService) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { - return nil, status.Error(codes.Unimplemented, "NodeGetVolumeStats is not implemented yet") + + if len(req.VolumeId) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume ID was empty") + } + if len(req.VolumePath) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume path was empty") + } + + exists, err := d.mounter.ExistsPath(req.VolumePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "unknown error when stat on %s: %v", req.VolumePath, err) + } + if !exists { + return nil, status.Errorf(codes.NotFound, "path %s does not exist", req.VolumePath) + } + + isBlock, err := isBlockDevice(req.VolumePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to determine whether %s is block device: %v", req.VolumePath, err) + } + if isBlock { + bcap, err := d.getBlockSizeBytes(req.VolumePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get block capacity on path %s: %v", req.VolumePath, err) + } + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Unit: csi.VolumeUsage_BYTES, + Total: bcap, + }, + }, + }, nil + } + + metrics, err := volume.NewMetricsStatFS(req.VolumePath).GetMetrics() + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not get metrics for %s: %s", req.VolumePath, err) + } + + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Total: metrics.Capacity.Value(), + Used: metrics.Used.Value(), + Available: metrics.Available.Value(), + Unit: csi.VolumeUsage_BYTES, + }, + + { + Total: metrics.Inodes.Value(), + Used: metrics.InodesUsed.Value(), + Available: metrics.InodesFree.Value(), + Unit: csi.VolumeUsage_INODES, + }, + }, + }, nil + } func (d *nodeService) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { @@ -550,3 +609,26 @@ func hasMountOption(options []string, opt string) bool { } return false } + +func (d *nodeService) getBlockSizeBytes(devicePath string) (int64, error) { + output, err := d.mounter.Run("blockdev", "--getsize64", devicePath) + if err != nil { + return -1, fmt.Errorf("error when getting size of block volume at path %s: output: %s, err: %v", devicePath, string(output), err) + } + strOut := strings.TrimSpace(string(output)) + gotSizeBytes, err := strconv.ParseInt(strOut, 10, 64) + if err != nil { + return -1, fmt.Errorf("failed to parse size %s into int a size", strOut) + } + return gotSizeBytes, nil +} + +func isBlockDevice(fullPath string) (bool, error) { + var st unix.Stat_t + err := unix.Stat(fullPath, &st) + if err != nil { + return false, err + } + + return (st.Mode & unix.S_IFMT) == unix.S_IFBLK, nil +} From 33985e103542b4aed99805b1f985d4fb53839a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Tue, 26 Nov 2019 13:04:31 -0800 Subject: [PATCH 2/4] Fix unit tests --- pkg/driver/node.go | 33 ++++++----------- pkg/driver/node_test.go | 22 ++++++------ pkg/driver/statter.go | 80 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 34 deletions(-) create mode 100644 pkg/driver/statter.go diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 7310c20d9..5a829d8cc 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -19,7 +19,6 @@ package driver import ( "context" "fmt" - "golang.org/x/sys/unix" "os" "path/filepath" "regexp" @@ -74,6 +73,7 @@ var ( type nodeService struct { metadata cloud.MetadataService mounter Mounter + statter Statter inFlight *internal.InFlight driverOptions *DriverOptions } @@ -89,6 +89,7 @@ func newNodeService(driverOptions *DriverOptions) nodeService { return nodeService{ metadata: metadata, mounter: newNodeMounter(), + statter: NewStatter(), inFlight: internal.NewInFlight(), driverOptions: driverOptions, } @@ -362,7 +363,7 @@ func (d *nodeService) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVo return nil, status.Errorf(codes.NotFound, "path %s does not exist", req.VolumePath) } - isBlock, err := isBlockDevice(req.VolumePath) + isBlock, err := d.statter.IsBlockDevice(req.VolumePath) if err != nil { return nil, status.Errorf(codes.Internal, "failed to determine whether %s is block device: %v", req.VolumePath, err) } @@ -381,29 +382,27 @@ func (d *nodeService) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVo }, nil } - metrics, err := volume.NewMetricsStatFS(req.VolumePath).GetMetrics() + available, capacity, used, inodesFree, inodes, inodesUsed, err := d.statter.StatFS(req.VolumePath) if err != nil { - return nil, status.Errorf(codes.Internal, "Could not get metrics for %s: %s", req.VolumePath, err) + return nil, status.Errorf(codes.Internal, "failed to get fs info on path %s: %v", req.VolumePath, err) } return &csi.NodeGetVolumeStatsResponse{ Usage: []*csi.VolumeUsage{ { - Total: metrics.Capacity.Value(), - Used: metrics.Used.Value(), - Available: metrics.Available.Value(), Unit: csi.VolumeUsage_BYTES, + Available: available, + Total: capacity, + Used: used, }, - { - Total: metrics.Inodes.Value(), - Used: metrics.InodesUsed.Value(), - Available: metrics.InodesFree.Value(), Unit: csi.VolumeUsage_INODES, + Available: inodesFree, + Total: inodes, + Used: inodesUsed, }, }, }, nil - } func (d *nodeService) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { @@ -622,13 +621,3 @@ func (d *nodeService) getBlockSizeBytes(devicePath string) (int64, error) { } return gotSizeBytes, nil } - -func isBlockDevice(fullPath string) (bool, error) { - var st unix.Stat_t - err := unix.Stat(fullPath, &st) - if err != nil { - return false, err - } - - return (st.Mode & unix.S_IFMT) == unix.S_IFBLK, nil -} diff --git a/pkg/driver/node_test.go b/pkg/driver/node_test.go index 7430d5fd1..81b219639 100644 --- a/pkg/driver/node_test.go +++ b/pkg/driver/node_test.go @@ -1175,26 +1175,24 @@ func TestNodeGetVolumeStats(t *testing.T) { mockMetadata := mocks.NewMockMetadataService(mockCtl) mockMounter := mocks.NewMockMounter(mockCtl) + VolumePath := "/test" + + mockMounter.EXPECT().ExistsPath(VolumePath).Return(true, nil) awsDriver := nodeService{ metadata: mockMetadata, mounter: mockMounter, + statter: NewFakeStatter(), inFlight: internal.NewInFlight(), } - expErrCode := codes.Unimplemented - - req := &csi.NodeGetVolumeStatsRequest{} - _, err := awsDriver.NodeGetVolumeStats(context.TODO(), req) - if err == nil { - t.Fatalf("Expected error code %d, got nil", expErrCode) + req := &csi.NodeGetVolumeStatsRequest{ + VolumeId: "vol-test", + VolumePath: VolumePath, } - srvErr, ok := status.FromError(err) - if !ok { - t.Fatalf("Could not get error status code from error: %v", srvErr) - } - if srvErr.Code() != expErrCode { - t.Fatalf("Expected error code %d, got %d message %s", expErrCode, srvErr.Code(), srvErr.Message()) + _, err := awsDriver.NodeGetVolumeStats(context.TODO(), req) + if err != nil { + t.Fatalf("Expected no error, got %v", err) } } diff --git a/pkg/driver/statter.go b/pkg/driver/statter.go new file mode 100644 index 000000000..439db372f --- /dev/null +++ b/pkg/driver/statter.go @@ -0,0 +1,80 @@ +/* +Copyright 2019 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +type Statter interface { + StatFS(path string) (int64, int64, int64, int64, int64, int64, error) + IsBlockDevice(string) (bool, error) +} + +var _ Statter = realStatter{} + +type realStatter struct { +} + +func NewStatter() realStatter { + return realStatter{} +} + +// IsBlock checks if the given path is a block device +func (realStatter) IsBlockDevice(fullPath string) (bool, error) { + var st unix.Stat_t + err := unix.Stat(fullPath, &st) + if err != nil { + return false, err + } + + return (st.Mode & unix.S_IFMT) == unix.S_IFBLK, nil +} + +func (realStatter) StatFS(path string) (available, capacity, used, inodesFree, inodes, inodesUsed int64, err error) { + statfs := &unix.Statfs_t{} + err = unix.Statfs(path, statfs) + if err != nil { + err = fmt.Errorf("failed to get fs info on path %s: %v", path, err) + return + } + + // Available is blocks available * fragment size + available = int64(statfs.Bavail) * int64(statfs.Bsize) + // Capacity is total block count * fragment size + capacity = int64(statfs.Blocks) * int64(statfs.Bsize) + // Usage is block being used * fragment size (aka block size). + used = (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize) + inodes = int64(statfs.Files) + inodesFree = int64(statfs.Ffree) + inodesUsed = inodes - inodesFree + return +} + +type fakeStatter struct{} + +func NewFakeStatter() fakeStatter { + return fakeStatter{} +} + +func (fakeStatter) StatFS(path string) (available, capacity, used, inodesFree, inodes, inodesUsed int64, err error) { + // Assume the file exists and give some dummy values back + return 1, 1, 1, 1, 1, 1, nil +} + +func (fakeStatter) IsBlockDevice(fullPath string) (bool, error) { + return false, nil +} From 7ffa093502eb010de6461e515c64ea01ee830810 Mon Sep 17 00:00:00 2001 From: "hendrik.leppelsack" Date: Tue, 16 Jun 2020 10:41:36 +0200 Subject: [PATCH 3/4] expose capability, fix, and log metrics --- go.mod | 1 + pkg/driver/node.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7887e75fd..6ceca21cd 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/onsi/gomega v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect golang.org/x/net v0.0.0-20200226051749-491c5fce7268 // indirect + golang.org/x/sys v0.0.0-20191220220014-0732a990476f google.golang.org/grpc v1.26.0 k8s.io/api v0.17.3 k8s.io/apimachinery v0.17.3 diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 5a829d8cc..b7a911c17 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -66,6 +66,7 @@ var ( nodeCaps = []csi.NodeServiceCapability_RPC_Type{ csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, csi.NodeServiceCapability_RPC_EXPAND_VOLUME, + csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, } ) @@ -347,6 +348,7 @@ func (d *nodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } func (d *nodeService) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + klog.V(4).Infof("NodeGetVolumeStats: called with args %+v", *req) if len(req.VolumeId) == 0 { return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume ID was empty") @@ -610,7 +612,8 @@ func hasMountOption(options []string, opt string) bool { } func (d *nodeService) getBlockSizeBytes(devicePath string) (int64, error) { - output, err := d.mounter.Run("blockdev", "--getsize64", devicePath) + cmd := d.mounter.Command("blockdev", "--getsize64", devicePath) + output, err := cmd.Output() if err != nil { return -1, fmt.Errorf("error when getting size of block volume at path %s: output: %s, err: %v", devicePath, string(output), err) } From 0f28898944087751cf098cf52e046324fb847bca Mon Sep 17 00:00:00 2001 From: Matthias Lee Date: Thu, 15 Oct 2020 13:25:39 -0400 Subject: [PATCH 4/4] update expected test response --- pkg/driver/node_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/driver/node_test.go b/pkg/driver/node_test.go index 81b219639..fa05b579f 100644 --- a/pkg/driver/node_test.go +++ b/pkg/driver/node_test.go @@ -1217,13 +1217,20 @@ func TestNodeGetCapabilities(t *testing.T) { }, }, }, - { + { Type: &csi.NodeServiceCapability_Rpc{ Rpc: &csi.NodeServiceCapability_RPC{ Type: csi.NodeServiceCapability_RPC_EXPAND_VOLUME, }, }, }, + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, + }, + }, + }, } expResp := &csi.NodeGetCapabilitiesResponse{Capabilities: caps}