feat(inputs.docker): Add disk usage

plugins/inputs/docker/README.md

@@ -53,6 +53,11 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
   # container_state_include = []
   # container_state_exclude = []
 
+  ## Objects to include for disk usage query
+  ## Allowed values are "container", "image", "volume" 
+  ## When empty disk usage is excluded
+  storage_objects = []
+
   ## Timeout for docker list, info, and stats commands
   timeout = "5s"
 
@@ -379,6 +384,23 @@ status if configured.
     - tasks_desired
     - tasks_running
 
+- docker_disk_usage
+  - tags:
+    - engine_host
+    - server_version
+    - container_name
+    - container_image
+    - container_version
+    - image_id
+    - image_name
+    - image_version
+    - volume_name
+  - fields:
+    - size_rw
+    - size_root_fs
+    - size
+    - shared_size
+
 ## Example Output
 
 ```text

plugins/inputs/docker/client.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	version        = "1.24" // https://docs.docker.com/engine/api/
+	version        = "1.43" // https://docs.docker.com/engine/api/
 	defaultHeaders = map[string]string{"User-Agent": "engine-api-cli-1.0"}
 )
 
@@ -23,6 +23,7 @@ type Client interface {
 	ServiceList(ctx context.Context, options types.ServiceListOptions) ([]swarm.Service, error)
 	TaskList(ctx context.Context, options types.TaskListOptions) ([]swarm.Task, error)
 	NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error)
+	DiskUsage(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error)
 	Close() error
 }
 
@@ -77,6 +78,9 @@ func (c *SocketClient) TaskList(ctx context.Context, options types.TaskListOptio
 func (c *SocketClient) NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error) {
 	return c.client.NodeList(ctx, options)
 }
+func (c *SocketClient) DiskUsage(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error) {
+	return c.client.DiskUsage(ctx, options)
+}
 func (c *SocketClient) Close() error {
 	return c.client.Close()
 }

plugins/inputs/docker/docker.go

@@ -53,6 +53,8 @@ type Docker struct {
 	ContainerStateInclude []string `toml:"container_state_include"`
 	ContainerStateExclude []string `toml:"container_state_exclude"`
 
+	StorageObjects []string `toml:"storage_objects"`
+
 	IncludeSourceTag bool `toml:"source_tag"`
 
 	Log telegraf.Logger
@@ -209,6 +211,27 @@ func (d *Docker) Gather(acc telegraf.Accumulator) error {
 	}
 	wg.Wait()
 
+	// Get disk usage data
+	if len(d.StorageObjects) > 0 {
+		objectTypes := []types.DiskUsageObject{}
+
+		for _, object := range d.StorageObjects {
+			if object == "container" {
+				objectTypes = append(objectTypes, types.ContainerObject)
+			} else if object == "image" {
+				objectTypes = append(objectTypes, types.ImageObject)
+			} else if object == "volume" {
+				objectTypes = append(objectTypes, types.VolumeObject)
+				// } else if object == "build-cache" {
+				// 	du_opts.Types = append(du_opts.Types, types.BuildCacheObject)
+			} else {
+				d.Log.Warnf("Unrecognized storage object type: %s", object)
+			}
+		}
+
+		d.gatherDiskUsage(acc, types.DiskUsageOptions{Types: objectTypes})
+	}
+
 	return nil
 }
 
@@ -412,21 +435,27 @@ func hostnameFromID(id string) string {
 	return id
 }
 
-func (d *Docker) gatherContainer(
-	container types.Container,
-	acc telegraf.Accumulator,
-) error {
-	var v *types.StatsJSON
-
-	// Parse container name
+// Parse container name
+func parseContainerName(containerNames []string) string {
 	var cname string
-	for _, name := range container.Names {
+
+	for _, name := range containerNames {
 		trimmedName := strings.TrimPrefix(name, "/")
 		if !strings.Contains(trimmedName, "/") {
 			cname = trimmedName
-			break
+			return cname
 		}
 	}
+	return cname
+}
+
+func (d *Docker) gatherContainer(
+	container types.Container,
+	acc telegraf.Accumulator,
+) error {
+	var v *types.StatsJSON
+
+	cname := parseContainerName(container.Names)
 
 	if cname == "" {
 		return nil
@@ -849,6 +878,97 @@ func (d *Docker) gatherBlockIOMetrics(
 	}
 }
 
+func (d *Docker) gatherDiskUsage(acc telegraf.Accumulator, opts types.DiskUsageOptions) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(d.Timeout))
+	defer cancel()
+
+	du, err := d.client.DiskUsage(ctx, opts)
+
+	if err != nil {
+		acc.AddError(err)
+	}
+
+	now := time.Now()
+	duName := "docker_disk_usage"
+
+	// Layers size
+	fields := map[string]interface{}{
+		"layers_size": du.LayersSize,
+	}
+
+	tags := map[string]string{
+		"engine_host":    d.engineHost,
+		"server_version": d.serverVersion,
+	}
+
+	acc.AddFields(duName, fields, tags, now)
+
+	// Containers
+	for _, container := range du.Containers {
+		fields := map[string]interface{}{
+			"size_rw":      container.SizeRw,
+			"size_root_fs": container.SizeRootFs,
+		}
+
+		imageName, imageVersion := dockerint.ParseImage(container.Image)
+
+		tags := map[string]string{
+			"engine_host":       d.engineHost,
+			"server_version":    d.serverVersion,
+			"container_name":    parseContainerName(container.Names),
+			"container_image":   imageName,
+			"container_version": imageVersion,
+		}
+
+		if d.IncludeSourceTag {
+			tags["source"] = hostnameFromID(container.ID)
+		}
+
+		acc.AddFields(duName, fields, tags, now)
+	}
+
+	// Images
+	for _, image := range du.Images {
+		fields := map[string]interface{}{
+			"size": image.Size,
+			// "virtual_size": image.VirtualSize, // deprecated
+			"shared_size": image.SharedSize,
+		}
+
+		tags := map[string]string{
+			"engine_host":    d.engineHost,
+			"server_version": d.serverVersion,
+			"image_id":       image.ID[7:19], // remove "sha256:" and keep the first 12 characters
+		}
+
+		// TODO: how to handle multiple repo tags?
+		if len(image.RepoTags) > 0 {
+			imageName, imageVersion := dockerint.ParseImage(image.RepoTags[0])
+			tags["image_name"] = imageName
+			tags["image_version"] = imageVersion
+		}
+
+		acc.AddFields(duName, fields, tags, now)
+	}
+
+	// Volumes
+	for _, volume := range du.Volumes {
+		fields := map[string]interface{}{
+			"size": volume.UsageData.Size,
+		}
+
+		tags := map[string]string{
+			"engine_host":    d.engineHost,
+			"server_version": d.serverVersion,
+			"volume_name":    volume.Name, // TODO: limit to first 12 characters if hash?
+		}
+
+		acc.AddFields(duName, fields, tags, now)
+	}
+
+	// Build Cache is not implemented
+}
+
 func copyTags(in map[string]string) map[string]string {
 	out := make(map[string]string)
 	for k, v := range in {

plugins/inputs/docker/docker_test.go

@@ -27,6 +27,7 @@ type MockClient struct {
 	ServiceListF      func(ctx context.Context, options types.ServiceListOptions) ([]swarm.Service, error)
 	TaskListF         func(ctx context.Context, options types.TaskListOptions) ([]swarm.Task, error)
 	NodeListF         func(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error)
+	DiskUsageF        func(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error)
 	CloseF            func() error
 }
 
@@ -77,6 +78,13 @@ func (c *MockClient) NodeList(
 	return c.NodeListF(ctx, options)
 }
 
+func (c *MockClient) DiskUsage(
+	ctx context.Context,
+	options types.DiskUsageOptions,
+) (types.DiskUsage, error) {
+	return c.DiskUsageF(ctx, options)
+}
+
 func (c *MockClient) Close() error {
 	return c.CloseF()
 }
@@ -103,6 +111,9 @@ var baseClient = MockClient{
 	NodeListF: func(context.Context, types.NodeListOptions) ([]swarm.Node, error) {
 		return NodeList, nil
 	},
+	DiskUsageF: func(context.Context, types.DiskUsageOptions) (types.DiskUsage, error) {
+		return DiskUsage, nil
+	},
 	CloseF: func() error {
 		return nil
 	},
@@ -445,6 +456,9 @@ func TestDocker_WindowsMemoryContainerStats(t *testing.T) {
 				NodeListF: func(context.Context, types.NodeListOptions) ([]swarm.Node, error) {
 					return NodeList, nil
 				},
+				DiskUsageF: func(context.Context, types.DiskUsageOptions) (types.DiskUsage, error) {
+					return DiskUsage, nil
+				},
 				CloseF: func() error {
 					return nil
 				},
@@ -1537,3 +1551,92 @@ func TestDocker_Init(t *testing.T) {
 		})
 	}
 }
+
+func TestDockerGatherDiskUsage(t *testing.T) {
+	var acc testutil.Accumulator
+	d := Docker{
+		Log:       testutil.Logger{},
+		newClient: func(string, *tls.Config) (Client, error) { return &baseClient, nil },
+	}
+
+	err := acc.GatherError(d.Gather)
+	require.NoError(t, err)
+
+	duOpts := types.DiskUsageOptions{Types: []types.DiskUsageObject{}}
+	d.gatherDiskUsage(&acc, duOpts)
+
+	acc.AssertContainsTaggedFields(t,
+		"docker_disk_usage",
+
+		map[string]interface{}{
+			"layers_size": int64(1e10),
+		},
+
+		map[string]string{
+			"engine_host":    "absol",
+			"server_version": "17.09.0-ce",
+		},
+	)
+
+	acc.AssertContainsTaggedFields(t,
+		"docker_disk_usage",
+
+		map[string]interface{}{
+			"size_root_fs": int64(123456789),
+			"size_rw":      int64(0)},
+
+		map[string]string{
+			"container_image":   "some_image",
+			"container_version": "1.0.0-alpine",
+			"engine_host":       "absol",
+			"server_version":    "17.09.0-ce",
+			"container_name":    "some_container",
+		},
+	)
+
+	acc.AssertContainsTaggedFields(t,
+		"docker_disk_usage",
+
+		map[string]interface{}{
+			"size":        int64(123456789),
+			"shared_size": int64(0)},
+
+		map[string]string{
+			"image_id":       "some_imageid",
+			"image_name":     "some_image_tag",
+			"image_version":  "1.0.0-alpine",
+			"engine_host":    "absol",
+			"server_version": "17.09.0-ce",
+		},
+	)
+
+	acc.AssertContainsTaggedFields(t,
+		"docker_disk_usage",
+
+		map[string]interface{}{
+			"size":        int64(425484494),
+			"shared_size": int64(0)},
+
+		map[string]string{
+			"image_id":       "7f4a1cc74046",
+			"image_name":     "telegraf",
+			"image_version":  "latest",
+			"engine_host":    "absol",
+			"server_version": "17.09.0-ce",
+		},
+	)
+
+	acc.AssertContainsTaggedFields(t,
+		"docker_disk_usage",
+
+		map[string]interface{}{
+			"size": int64(123456789),
+		},
+
+		map[string]string{
+			"volume_name":    "some_volume",
+			"engine_host":    "absol",
+			"server_version": "17.09.0-ce",
+		},
+	)
+}

plugins/inputs/docker/docker_testdata.go

@@ -10,6 +10,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/volume"
 )
 
 var info = types.Info{
@@ -542,3 +543,15 @@ func containerInspect() types.ContainerJSON {
 		},
 	}
 }
+
+var DiskUsage = types.DiskUsage{
+	LayersSize: 1e10,
+	Containers: []*types.Container{
+		{Names: []string{"/some_container"}, Image: "some_image:1.0.0-alpine", SizeRw: 0, SizeRootFs: 123456789},
+	},
+	Images: []*types.ImageSummary{
+		{ID: "sha256:some_imageid", RepoTags: []string{"some_image_tag:1.0.0-alpine"}, Size: 123456789, SharedSize: 0},
+		{ID: "sha256:7f4a1cc74046ce48cd918693cd6bf4b2683f4ce0d7be3f7148a21df9f06f5b5f", RepoTags: []string{"telegraf:latest"}, Size: 425484494, SharedSize: 0},
+	},
+	Volumes: []*volume.Volume{{Name: "some_volume", UsageData: &volume.UsageData{Size: 123456789}}},
+}