From 1a9faced990ce8bca49bf94cb6ed406e9d4cea51 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 11 Nov 2024 15:51:03 -0500 Subject: [PATCH] dynamic host volumes: basic CLI CRUD operations (#24382) This changeset implements a first pass at the CLI for Dynamic Host Volumes. Ref: https://hashicorp.atlassian.net/browse/NET-11549 --- api/host_volumes.go | 236 ++++++++++++++++++ command/volume_create.go | 10 +- command/volume_create_host.go | 228 +++++++++++++++++ command/volume_create_host_test.go | 227 +++++++++++++++++ command/volume_delete.go | 31 ++- command/volume_delete_host_test.go | 75 ++++++ command/volume_register.go | 14 +- ...er_test.go => volume_register_csi_test.go} | 0 command/volume_register_host.go | 35 +++ command/volume_register_host_test.go | 93 +++++++ command/volume_status.go | 33 ++- command/volume_status_csi.go | 8 +- ...atus_test.go => volume_status_csi_test.go} | 0 command/volume_status_host.go | 180 +++++++++++++ command/volume_status_host_test.go | 150 +++++++++++ 15 files changed, 1295 insertions(+), 25 deletions(-) create mode 100644 api/host_volumes.go create mode 100644 command/volume_create_host.go create mode 100644 command/volume_create_host_test.go create mode 100644 command/volume_delete_host_test.go rename command/{volume_register_test.go => volume_register_csi_test.go} (100%) create mode 100644 command/volume_register_host.go create mode 100644 command/volume_register_host_test.go rename command/{volume_status_test.go => volume_status_csi_test.go} (100%) create mode 100644 command/volume_status_host.go create mode 100644 command/volume_status_host_test.go diff --git a/api/host_volumes.go b/api/host_volumes.go new file mode 100644 index 00000000000..dae11afc68a --- /dev/null +++ b/api/host_volumes.go @@ -0,0 +1,236 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package api + +import "net/url" + +// HostVolume represents a Dynamic Host Volume: a volume associated with a +// specific Nomad client agent but created via API. +type HostVolume struct { + // Namespace is the Nomad namespace for the host volume, which constrains + // which jobs can mount it. + Namespace string `mapstructure:"namespace" hcl:"namespace"` + + // ID is a UUID-like string generated by the server. + ID string `mapstructure:"id" hcl:"id"` + + // Name is the name that group.volume will use to identify the volume + // source. Not expected to be unique. + Name string `mapstructure:"name" hcl:"name"` + + // PluginID is the name of the host volume plugin on the client that will be + // used for creating the volume. If omitted, the client will use its default + // built-in plugin. + PluginID string `mapstructure:"plugin_id" hcl:"plugin_id"` + + // NodePool is the node pool of the node where the volume is placed. If the + // user doesn't provide a node ID, a node will be selected using the + // NodePool and Constraints. If the user provides both NodePool and NodeID, + // NodePool will be used to validate the request. If omitted, the server + // will populate this value in before writing the volume to Raft. + NodePool string `mapstructure:"node_pool" hcl:"node_pool"` + + // NodeID is the node where the volume is placed. If the user doesn't + // provide a NodeID, one will be selected using the NodePool and + // Constraints. If omitted, this field will then be populated by the server + // before writing the volume to Raft. + NodeID string `mapstructure:"node_id" hcl:"node_id"` + + // Constraints are optional. If the NodeID is not provided, the NodePool and + // Constraints are used to select a node. If the NodeID is provided, + // Constraints are used to validate that the node meets those constraints at + // the time of volume creation. + Constraints []*Constraint `json:",omitempty" hcl:"constraint"` + + // Because storage may allow only specific intervals of size, we accept a + // min and max and return the actual capacity when the volume is created or + // updated on the client + RequestedCapacityMinBytes int64 `mapstructure:"capacity_min" hcl:"capacity_min"` + RequestedCapacityMaxBytes int64 `mapstructure:"capacity_max" hcl:"capacity_max"` + CapacityBytes int64 + + // RequestedCapabilities defines the options available to group.volume + // blocks. The scheduler checks against the listed capability blocks and + // selects a node for placement if *any* capability block works. + RequestedCapabilities []*HostVolumeCapability `hcl:"capability"` + + // Parameters are an opaque map of parameters for the host volume plugin. + Parameters map[string]string `json:",omitempty"` + + // HostPath is the path on disk where the volume's mount point was + // created. We record this to make debugging easier. + HostPath string `mapstructure:"host_path" hcl:"host_path"` + + // State represents the overall state of the volume. One of pending, ready, + // deleted. + State HostVolumeState + + CreateIndex uint64 + CreateTime int64 + + ModifyIndex uint64 + ModifyTime int64 + + // Allocations is the list of non-client-terminal allocations with claims on + // this host volume. They are denormalized on read and this field will be + // never written to Raft + Allocations []*AllocationListStub `json:",omitempty" mapstructure:"-" hcl:"-"` +} + +// HostVolume state reports the current status of the host volume +type HostVolumeState string + +const ( + HostVolumeStatePending HostVolumeState = "pending" + HostVolumeStateReady HostVolumeState = "ready" + HostVolumeStateDeleted HostVolumeState = "deleted" +) + +// HostVolumeCapability is the requested attachment and access mode for a volume +type HostVolumeCapability struct { + AttachmentMode HostVolumeAttachmentMode `mapstructure:"attachment_mode" hcl:"attachment_mode"` + AccessMode HostVolumeAccessMode `mapstructure:"access_mode" hcl:"access_mode"` +} + +// HostVolumeAttachmentMode chooses the type of storage API that will be used to +// interact with the device. +type HostVolumeAttachmentMode string + +const ( + HostVolumeAttachmentModeUnknown HostVolumeAttachmentMode = "" + HostVolumeAttachmentModeBlockDevice HostVolumeAttachmentMode = "block-device" + HostVolumeAttachmentModeFilesystem HostVolumeAttachmentMode = "file-system" +) + +// HostVolumeAccessMode indicates how Nomad should make the volume available to +// concurrent allocations. +type HostVolumeAccessMode string + +const ( + HostVolumeAccessModeUnknown HostVolumeAccessMode = "" + + HostVolumeAccessModeSingleNodeReader HostVolumeAccessMode = "single-node-reader-only" + HostVolumeAccessModeSingleNodeWriter HostVolumeAccessMode = "single-node-writer" + + HostVolumeAccessModeMultiNodeReader HostVolumeAccessMode = "multi-node-reader-only" + HostVolumeAccessModeMultiNodeSingleWriter HostVolumeAccessMode = "multi-node-single-writer" + HostVolumeAccessModeMultiNodeMultiWriter HostVolumeAccessMode = "multi-node-multi-writer" +) + +// HostVolumeStub is used for responses for the List Volumes endpoint +type HostVolumeStub struct { + Namespace string + ID string + Name string + PluginID string + NodePool string + NodeID string + CapacityBytes int64 + State HostVolumeState + + CreateIndex uint64 + CreateTime int64 + + ModifyIndex uint64 + ModifyTime int64 +} + +// HostVolumes is used to access the host volumes API. +type HostVolumes struct { + client *Client +} + +// HostVolumes returns a new handle on the host volumes API. +func (c *Client) HostVolumes() *HostVolumes { + return &HostVolumes{client: c} +} + +type HostVolumeCreateRequest struct { + Volumes []*HostVolume +} + +type HostVolumeRegisterRequest struct { + Volumes []*HostVolume +} + +type HostVolumeListRequest struct { + NodeID string + NodePool string +} + +type HostVolumeDeleteRequest struct { + VolumeIDs []string +} + +// Create forwards to client agents so host volumes can be created on those +// hosts, and registers the volumes with Nomad servers. +func (hv *HostVolumes) Create(req *HostVolumeCreateRequest, opts *WriteOptions) ([]*HostVolume, *WriteMeta, error) { + var out struct { + Volumes []*HostVolume + } + wm, err := hv.client.put("/v1/volume/host/create", req, &out, opts) + if err != nil { + return nil, wm, err + } + return out.Volumes, wm, nil +} + +// Register registers host volumes that were created out-of-band with the Nomad +// servers. +func (hv *HostVolumes) Register(req *HostVolumeRegisterRequest, opts *WriteOptions) ([]*HostVolume, *WriteMeta, error) { + var out struct { + Volumes []*HostVolume + } + wm, err := hv.client.put("/v1/volume/host/register", req, &out, opts) + if err != nil { + return nil, wm, err + } + return out.Volumes, wm, nil +} + +// Get queries for a single host volume, by ID +func (hv *HostVolumes) Get(id string, opts *QueryOptions) (*HostVolume, *QueryMeta, error) { + var out *HostVolume + path, err := url.JoinPath("/v1/volume/host/", url.PathEscape(id)) + if err != nil { + return nil, nil, err + } + qm, err := hv.client.query(path, &out, opts) + if err != nil { + return nil, qm, err + } + return out, qm, nil +} + +// List queries for a set of host volumes, by namespace, node, node pool, or +// name prefix. +func (hv *HostVolumes) List(req *HostVolumeListRequest, opts *QueryOptions) ([]*HostVolumeStub, *QueryMeta, error) { + var out []*HostVolumeStub + qv := url.Values{} + qv.Set("type", "host") + if req != nil { + if req.NodeID != "" { + qv.Set("node_id", req.NodeID) + } + if req.NodePool != "" { + qv.Set("node_pool", req.NodePool) + } + } + + qm, err := hv.client.query("/v1/volumes?"+qv.Encode(), &out, opts) + if err != nil { + return nil, qm, err + } + return out, qm, nil +} + +// Delete deletes a host volume +func (hv *HostVolumes) Delete(id string, opts *WriteOptions) (*WriteMeta, error) { + path, err := url.JoinPath("/v1/volume/host/", url.PathEscape(id)) + if err != nil { + return nil, err + } + wm, err := hv.client.delete(path, nil, nil, opts) + return wm, err +} diff --git a/command/volume_create.go b/command/volume_create.go index c7d32fbe808..258e37b1f47 100644 --- a/command/volume_create.go +++ b/command/volume_create.go @@ -25,8 +25,9 @@ Usage: nomad volume create [options] If the supplied path is "-" the volume file is read from stdin. Otherwise, it is read from the file at the supplied path. - When ACLs are enabled, this command requires a token with the - 'csi-write-volume' capability for the volume's namespace. + When ACLs are enabled, this command requires a token with the appropriate + capability in the volume's namespace: the 'csi-write-volume' capability for + CSI volumes or 'host-volume-create' for dynamic host volumes. General Options: @@ -99,8 +100,9 @@ func (c *VolumeCreateCommand) Run(args []string) int { switch strings.ToLower(volType) { case "csi": - code := c.csiCreate(client, ast) - return code + return c.csiCreate(client, ast) + case "host": + return c.hostVolumeCreate(client, ast) default: c.Ui.Error(fmt.Sprintf("Error unknown volume type: %s", volType)) return 1 diff --git a/command/volume_create_host.go b/command/volume_create_host.go new file mode 100644 index 00000000000..32205610740 --- /dev/null +++ b/command/volume_create_host.go @@ -0,0 +1,228 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "strconv" + + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/mapstructure" +) + +func (c *VolumeCreateCommand) hostVolumeCreate(client *api.Client, ast *ast.File) int { + vol, err := decodeHostVolume(ast) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error decoding the volume definition: %s", err)) + return 1 + } + + req := &api.HostVolumeCreateRequest{ + Volumes: []*api.HostVolume{vol}, + } + vols, _, err := client.HostVolumes().Create(req, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error creating volume: %s", err)) + return 1 + } + for _, vol := range vols { + // note: the command only ever returns 1 volume from the API + c.Ui.Output(fmt.Sprintf( + "Created host volume %s with ID %s", vol.Name, vol.ID)) + } + + // TODO(1.10.0): monitor so we can report when the node has fingerprinted + + return 0 +} + +func decodeHostVolume(input *ast.File) (*api.HostVolume, error) { + var err error + vol := &api.HostVolume{} + + list, ok := input.Node.(*ast.ObjectList) + if !ok { + return nil, fmt.Errorf("error parsing: root should be an object") + } + + // Decode the full thing into a map[string]interface for ease + var m map[string]any + err = hcl.DecodeObject(&m, list) + if err != nil { + return nil, err + } + + // Need to manually parse these fields/blocks + delete(m, "capability") + delete(m, "constraint") + delete(m, "capacity_max") + delete(m, "capacity_min") + delete(m, "type") + + // Decode the rest + err = mapstructure.WeakDecode(m, vol) + if err != nil { + return nil, err + } + + capacityMin, err := parseCapacityBytes(list.Filter("capacity_min")) + if err != nil { + return nil, fmt.Errorf("invalid capacity_min: %v", err) + } + vol.RequestedCapacityMinBytes = capacityMin + capacityMax, err := parseCapacityBytes(list.Filter("capacity_max")) + if err != nil { + return nil, fmt.Errorf("invalid capacity_max: %v", err) + } + vol.RequestedCapacityMaxBytes = capacityMax + + if o := list.Filter("constraint"); len(o.Items) > 0 { + if err := parseConstraints(&vol.Constraints, o); err != nil { + return nil, fmt.Errorf("invalid constraint: %v", err) + } + } + if o := list.Filter("capability"); len(o.Items) > 0 { + if err := parseHostVolumeCapabilities(&vol.RequestedCapabilities, o); err != nil { + return nil, fmt.Errorf("invalid capability: %v", err) + } + } + + return vol, nil +} + +func parseHostVolumeCapabilities(result *[]*api.HostVolumeCapability, list *ast.ObjectList) error { + for _, o := range list.Elem().Items { + valid := []string{"access_mode", "attachment_mode"} + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return err + } + + ot, ok := o.Val.(*ast.ObjectType) + if !ok { + break + } + + var m map[string]any + if err := hcl.DecodeObject(&m, ot.List); err != nil { + return err + } + var cap *api.HostVolumeCapability + if err := mapstructure.WeakDecode(&m, &cap); err != nil { + return err + } + + *result = append(*result, cap) + } + + return nil +} + +func parseConstraints(result *[]*api.Constraint, list *ast.ObjectList) error { + for _, o := range list.Elem().Items { + valid := []string{ + "attribute", + "distinct_hosts", + "distinct_property", + "operator", + "regexp", + "set_contains", + "value", + "version", + "semver", + } + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return err + } + + var m map[string]any + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + m["LTarget"] = m["attribute"] + m["RTarget"] = m["value"] + m["Operand"] = m["operator"] + + // If "version" is provided, set the operand + // to "version" and the value to the "RTarget" + if constraint, ok := m[api.ConstraintVersion]; ok { + m["Operand"] = api.ConstraintVersion + m["RTarget"] = constraint + } + + // If "semver" is provided, set the operand + // to "semver" and the value to the "RTarget" + if constraint, ok := m[api.ConstraintSemver]; ok { + m["Operand"] = api.ConstraintSemver + m["RTarget"] = constraint + } + + // If "regexp" is provided, set the operand + // to "regexp" and the value to the "RTarget" + if constraint, ok := m[api.ConstraintRegex]; ok { + m["Operand"] = api.ConstraintRegex + m["RTarget"] = constraint + } + + // If "set_contains" is provided, set the operand + // to "set_contains" and the value to the "RTarget" + if constraint, ok := m[api.ConstraintSetContains]; ok { + m["Operand"] = api.ConstraintSetContains + m["RTarget"] = constraint + } + + if value, ok := m[api.ConstraintDistinctHosts]; ok { + enabled, err := parseBool(value) + if err != nil { + return fmt.Errorf("distinct_hosts should be set to true or false; %v", err) + } + + // If it is not enabled, skip the constraint. + if !enabled { + continue + } + + m["Operand"] = api.ConstraintDistinctHosts + m["RTarget"] = strconv.FormatBool(enabled) + } + + if property, ok := m[api.ConstraintDistinctProperty]; ok { + m["Operand"] = api.ConstraintDistinctProperty + m["LTarget"] = property + } + + // Build the constraint + var c api.Constraint + if err := mapstructure.WeakDecode(m, &c); err != nil { + return err + } + if c.Operand == "" { + c.Operand = "=" + } + + *result = append(*result, &c) + } + + return nil +} + +// parseBool takes an interface value and tries to convert it to a boolean and +// returns an error if the type can't be converted. +func parseBool(value any) (bool, error) { + var enabled bool + var err error + switch data := value.(type) { + case string: + enabled, err = strconv.ParseBool(data) + case bool: + enabled = data + default: + err = fmt.Errorf("%v couldn't be converted to boolean value", value) + } + + return enabled, err +} diff --git a/command/volume_create_host_test.go b/command/volume_create_host_test.go new file mode 100644 index 00000000000..81e59367ff1 --- /dev/null +++ b/command/volume_create_host_test.go @@ -0,0 +1,227 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "os" + "strings" + "testing" + + "github.com/hashicorp/hcl" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/shoenig/test/must" +) + +func TestHostVolumeCreateCommand_Run(t *testing.T) { + ci.Parallel(t) + srv, client, url := testServer(t, true, nil) + t.Cleanup(srv.Shutdown) + + waitForNodes(t, client) + + _, err := client.Namespaces().Register(&api.Namespace{Name: "prod"}, nil) + must.NoError(t, err) + + ui := cli.NewMockUi() + cmd := &VolumeCreateCommand{Meta: Meta{Ui: ui}} + + hclTestFile := ` +namespace = "prod" +name = "database" +type = "host" +plugin_id = "plugin_id" +node_pool = "default" + +capacity_min = "10GiB" +capacity_max = "20G" + +constraint { + attribute = "${attr.kernel.name}" + value = "linux" +} + +constraint { + attribute = "${meta.rack}" + value = "foo" +} + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +capability { + access_mode = "single-node-reader" + attachment_mode = "block-device" +} + +parameters { + foo = "bar" +} +` + + file, err := os.CreateTemp(t.TempDir(), "volume-test-*.hcl") + must.NoError(t, err) + _, err = file.WriteString(hclTestFile) + must.NoError(t, err) + + args := []string{"-address", url, file.Name()} + + code := cmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + + out := ui.OutputWriter.String() + must.StrContains(t, out, "Created host volume") + parts := strings.Split(out, " ") + id := strings.TrimSpace(parts[len(parts)-1]) + + // Verify volume was created + got, _, err := client.HostVolumes().Get(id, &api.QueryOptions{Namespace: "prod"}) + must.NoError(t, err) + must.NotNil(t, got) +} + +func TestHostVolume_HCLDecode(t *testing.T) { + ci.Parallel(t) + + cases := []struct { + name string + hcl string + expected *api.HostVolume + errMsg string + }{ + { + name: "full spec", + hcl: ` +namespace = "prod" +name = "database" +type = "host" +plugin_id = "plugin_id" +node_pool = "default" + +capacity_min = "10GiB" +capacity_max = "20G" + +constraint { + attribute = "${attr.kernel.name}" + value = "linux" +} + +constraint { + attribute = "${meta.rack}" + value = "foo" +} + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +capability { + access_mode = "single-node-reader-only" + attachment_mode = "block-device" +} + +parameters { + foo = "bar" +} +`, + expected: &api.HostVolume{ + Namespace: "prod", + Name: "database", + PluginID: "plugin_id", + NodePool: "default", + Constraints: []*api.Constraint{{ + LTarget: "${attr.kernel.name}", + RTarget: "linux", + Operand: "=", + }, { + LTarget: "${meta.rack}", + RTarget: "foo", + Operand: "=", + }}, + RequestedCapacityMinBytes: 10737418240, + RequestedCapacityMaxBytes: 20000000000, + RequestedCapabilities: []*api.HostVolumeCapability{ + { + AttachmentMode: api.HostVolumeAttachmentModeFilesystem, + AccessMode: api.HostVolumeAccessModeSingleNodeWriter, + }, + { + AttachmentMode: api.HostVolumeAttachmentModeBlockDevice, + AccessMode: api.HostVolumeAccessModeSingleNodeReader, + }, + }, + Parameters: map[string]string{"foo": "bar"}, + }, + }, + + { + name: "mostly empty spec", + hcl: ` +namespace = "prod" +name = "database" +type = "host" +plugin_id = "plugin_id" +node_pool = "default" +`, + expected: &api.HostVolume{ + Namespace: "prod", + Name: "database", + PluginID: "plugin_id", + NodePool: "default", + }, + }, + + { + name: "invalid capacity", + hcl: ` +namespace = "prod" +name = "database" +type = "host" +plugin_id = "plugin_id" +node_pool = "default" + +capacity_min = "a" +`, + expected: nil, + errMsg: "invalid capacity_min: could not parse value as bytes: strconv.ParseFloat: parsing \"\": invalid syntax", + }, + + { + name: "invalid constraint", + hcl: ` +namespace = "prod" +name = "database" +type = "host" +plugin_id = "plugin_id" +node_pool = "default" + +constraint { + distinct_hosts = "foo" +} + +`, + expected: nil, + errMsg: "invalid constraint: distinct_hosts should be set to true or false; strconv.ParseBool: parsing \"foo\": invalid syntax", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ast, err := hcl.ParseString(tc.hcl) + must.NoError(t, err) + vol, err := decodeHostVolume(ast) + if tc.errMsg == "" { + must.NoError(t, err) + } else { + must.EqError(t, err, tc.errMsg) + } + must.Eq(t, tc.expected, vol) + }) + } + +} diff --git a/command/volume_delete.go b/command/volume_delete.go index 7dc3df1e128..ab8be61104b 100644 --- a/command/volume_delete.go +++ b/command/volume_delete.go @@ -41,6 +41,9 @@ Delete Options: -secret Secrets to pass to the plugin to delete the snapshot. Accepts multiple flags in the form -secret key=value + + -type + Type of volume to delete. Must be one of "csi" or "host". Defaults to "csi". ` return strings.TrimSpace(helpText) } @@ -80,9 +83,11 @@ func (c *VolumeDeleteCommand) Name() string { return "volume delete" } func (c *VolumeDeleteCommand) Run(args []string) int { var secretsArgs flaghelper.StringFlag + var typeArg string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } flags.Var(&secretsArgs, "secret", "secrets for snapshot, ex. -secret key=value") + flags.StringVar(&typeArg, "type", "csi", "type of volume (csi or host)") if err := flags.Parse(args); err != nil { c.Ui.Error(fmt.Sprintf("Error parsing arguments %s", err)) @@ -105,6 +110,19 @@ func (c *VolumeDeleteCommand) Run(args []string) int { return 1 } + switch typeArg { + case "csi": + return c.deleteCSIVolume(client, volID, secretsArgs) + case "host": + return c.deleteHostVolume(client, volID) + default: + c.Ui.Error(fmt.Sprintf("No such volume type %q", typeArg)) + return 1 + } +} + +func (c *VolumeDeleteCommand) deleteCSIVolume(client *api.Client, volID string, secretsArgs flaghelper.StringFlag) int { + secrets := api.CSISecrets{} for _, kv := range secretsArgs { if key, value, found := strings.Cut(kv, "="); found { @@ -115,7 +133,7 @@ func (c *VolumeDeleteCommand) Run(args []string) int { } } - err = client.CSIVolumes().DeleteOpts(&api.CSIVolumeDeleteRequest{ + err := client.CSIVolumes().DeleteOpts(&api.CSIVolumeDeleteRequest{ ExternalVolumeID: volID, Secrets: secrets, }, nil) @@ -127,3 +145,14 @@ func (c *VolumeDeleteCommand) Run(args []string) int { c.Ui.Output(fmt.Sprintf("Successfully deleted volume %q!", volID)) return 0 } + +func (c *VolumeDeleteCommand) deleteHostVolume(client *api.Client, volID string) int { + _, err := client.HostVolumes().Delete(volID, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error deleting volume: %s", err)) + return 1 + } + + c.Ui.Output(fmt.Sprintf("Successfully deleted volume %q!", volID)) + return 0 +} diff --git a/command/volume_delete_host_test.go b/command/volume_delete_host_test.go new file mode 100644 index 00000000000..4da028d7085 --- /dev/null +++ b/command/volume_delete_host_test.go @@ -0,0 +1,75 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/shoenig/test/must" +) + +func TestHostVolumeDeleteCommand(t *testing.T) { + ci.Parallel(t) + srv, client, url := testServer(t, true, nil) + t.Cleanup(srv.Shutdown) + + waitForNodes(t, client) + + _, err := client.Namespaces().Register(&api.Namespace{Name: "prod"}, nil) + must.NoError(t, err) + + nodes, _, err := client.Nodes().List(nil) + must.NoError(t, err) + must.Len(t, 1, nodes) + nodeID := nodes[0].ID + + ui := cli.NewMockUi() + + hclTestFile := fmt.Sprintf(` +namespace = "prod" +name = "example" +type = "host" +plugin_id = "plugin_id" +node_id = "%s" +node_pool = "default" +`, nodeID) + + file, err := os.CreateTemp(t.TempDir(), "volume-test-*.hcl") + must.NoError(t, err) + _, err = file.WriteString(hclTestFile) + must.NoError(t, err) + + args := []string{"-address", url, file.Name()} + regCmd := &VolumeRegisterCommand{Meta: Meta{Ui: ui}} + code := regCmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + + out := ui.OutputWriter.String() + must.StrContains(t, out, "Registered host volume") + parts := strings.Split(out, " ") + id := strings.TrimSpace(parts[len(parts)-1]) + + ui.OutputWriter.Reset() + + // missing the namespace + cmd := &VolumeDeleteCommand{Meta: Meta{Ui: ui}} + args = []string{"-address", url, "-type", "host", id} + code = cmd.Run(args) + must.Eq(t, 1, code) + must.StrContains(t, ui.ErrorWriter.String(), "no such volume") + ui.ErrorWriter.Reset() + + // fix the namespace + args = []string{"-address", url, "-type", "host", "-namespace", "prod", id} + code = cmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + out = ui.OutputWriter.String() + must.StrContains(t, out, fmt.Sprintf("Successfully deleted volume %q!", id)) +} diff --git a/command/volume_register.go b/command/volume_register.go index 3a8815347ff..19527cf6b1f 100644 --- a/command/volume_register.go +++ b/command/volume_register.go @@ -28,8 +28,9 @@ Usage: nomad volume register [options] If the supplied path is "-" the volume file is read from stdin. Otherwise, it is read from the file at the supplied path. - When ACLs are enabled, this command requires a token with the - 'csi-write-volume' capability for the volume's namespace. + When ACLs are enabled, this command requires a token with the appropriate + capability in the volume's namespace: the 'csi-write-volume' capability for + CSI volumes or 'host-volume-register' for dynamic host volumes. General Options: @@ -103,16 +104,13 @@ func (c *VolumeRegisterCommand) Run(args []string) int { switch volType { case "csi": - code := c.csiRegister(client, ast) - if code != 0 { - return code - } + return c.csiRegister(client, ast) + case "host": + return c.hostVolumeRegister(client, ast) default: c.Ui.Error(fmt.Sprintf("Error unknown volume type: %s", volType)) return 1 } - - return 0 } // parseVolume is used to parse the quota specification from HCL diff --git a/command/volume_register_test.go b/command/volume_register_csi_test.go similarity index 100% rename from command/volume_register_test.go rename to command/volume_register_csi_test.go diff --git a/command/volume_register_host.go b/command/volume_register_host.go new file mode 100644 index 00000000000..705f2faaf26 --- /dev/null +++ b/command/volume_register_host.go @@ -0,0 +1,35 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" +) + +func (c *VolumeRegisterCommand) hostVolumeRegister(client *api.Client, ast *ast.File) int { + vol, err := decodeHostVolume(ast) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error decoding the volume definition: %s", err)) + return 1 + } + + req := &api.HostVolumeRegisterRequest{ + Volumes: []*api.HostVolume{vol}, + } + vols, _, err := client.HostVolumes().Register(req, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error registering volume: %s", err)) + return 1 + } + for _, vol := range vols { + // note: the command only ever returns 1 volume from the API + c.Ui.Output(fmt.Sprintf( + "Registered host volume %s with ID %s", vol.Name, vol.ID)) + } + + return 0 +} diff --git a/command/volume_register_host_test.go b/command/volume_register_host_test.go new file mode 100644 index 00000000000..0ce33770197 --- /dev/null +++ b/command/volume_register_host_test.go @@ -0,0 +1,93 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/shoenig/test/must" +) + +func TestHostVolumeRegisterCommand_Run(t *testing.T) { + ci.Parallel(t) + srv, client, url := testServer(t, true, nil) + t.Cleanup(srv.Shutdown) + + waitForNodes(t, client) + + _, err := client.Namespaces().Register(&api.Namespace{Name: "prod"}, nil) + must.NoError(t, err) + + nodes, _, err := client.Nodes().List(nil) + must.NoError(t, err) + must.Len(t, 1, nodes) + nodeID := nodes[0].ID + + ui := cli.NewMockUi() + cmd := &VolumeRegisterCommand{Meta: Meta{Ui: ui}} + + hclTestFile := fmt.Sprintf(` +namespace = "prod" +name = "database" +type = "host" +plugin_id = "plugin_id" +node_id = "%s" +node_pool = "default" + +capacity = 150000000 +host_path = "/var/nomad/alloc_mounts/example" +capacity_min = "10GiB" +capacity_max = "20G" + +constraint { + attribute = "${attr.kernel.name}" + value = "linux" +} + +constraint { + attribute = "${meta.rack}" + value = "foo" +} + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +capability { + access_mode = "single-node-reader-only" + attachment_mode = "block-device" +} + +parameters { + foo = "bar" +} +`, nodeID) + + file, err := os.CreateTemp(t.TempDir(), "volume-test-*.hcl") + must.NoError(t, err) + _, err = file.WriteString(hclTestFile) + must.NoError(t, err) + + args := []string{"-address", url, file.Name()} + + code := cmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + + out := ui.OutputWriter.String() + must.StrContains(t, out, "Registered host volume") + parts := strings.Split(out, " ") + id := strings.TrimSpace(parts[len(parts)-1]) + + // Verify volume was registered + got, _, err := client.HostVolumes().Get(id, &api.QueryOptions{Namespace: "prod"}) + must.NoError(t, err) + must.NotNil(t, got) +} diff --git a/command/volume_status.go b/command/volume_status.go index 22fc6afc225..b6163cca6f4 100644 --- a/command/volume_status.go +++ b/command/volume_status.go @@ -52,6 +52,12 @@ Status Options: -t Format and display volumes using a Go template. + + -node-pool + Filter results by node pool, when no volume ID is provided and -type=host. + + -node + Filter results by node ID, when no volume ID is provided and -type=host. ` return strings.TrimSpace(helpText) } @@ -68,6 +74,10 @@ func (c *VolumeStatusCommand) AutocompleteFlags() complete.Flags { "-verbose": complete.PredictNothing, "-json": complete.PredictNothing, "-t": complete.PredictAnything, + + // TODO(1.10.0): wire-up predictions for nodes and node pools + "-node": complete.PredictNothing, + "-node-pool": complete.PredictNothing, }) } @@ -89,7 +99,7 @@ func (c *VolumeStatusCommand) AutocompleteArgs() complete.Predictor { func (c *VolumeStatusCommand) Name() string { return "volume status" } func (c *VolumeStatusCommand) Run(args []string) int { - var typeArg string + var typeArg, nodeID, nodePool string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } @@ -98,6 +108,8 @@ func (c *VolumeStatusCommand) Run(args []string) int { flags.BoolVar(&c.verbose, "verbose", false, "") flags.BoolVar(&c.json, "json", false, "") flags.StringVar(&c.template, "t", "", "") + flags.StringVar(&nodeID, "node", "", "") + flags.StringVar(&nodePool, "node-pool", "", "") if err := flags.Parse(args); err != nil { c.Ui.Error(fmt.Sprintf("Error parsing arguments %s", err)) @@ -130,12 +142,17 @@ func (c *VolumeStatusCommand) Run(args []string) int { id = args[0] } - code := c.csiStatus(client, id) - if code != 0 { - return code + switch typeArg { + case "csi", "": + if nodeID != "" || nodePool != "" { + c.Ui.Error("-node and -node-pool can only be used with -type host") + return 1 + } + return c.csiStatus(client, id) + case "host": + return c.hostVolumeStatus(client, id, nodeID, nodePool) + default: + c.Ui.Error(fmt.Sprintf("No such volume type %q", typeArg)) + return 1 } - - // Extend this section with other volume implementations - - return 0 } diff --git a/command/volume_status_csi.go b/command/volume_status_csi.go index 31fdeeb2331..01644b513d8 100644 --- a/command/volume_status_csi.go +++ b/command/volume_status_csi.go @@ -23,7 +23,7 @@ func (c *VolumeStatusCommand) csiBanner() { func (c *VolumeStatusCommand) csiStatus(client *api.Client, id string) int { // Invoke list mode if no volume id if id == "" { - return c.listVolumes(client) + return c.listCSIVolumes(client) } // get a CSI volume that matches the given prefix or a list of all matches if an @@ -55,7 +55,7 @@ func (c *VolumeStatusCommand) csiStatus(client *api.Client, id string) int { return 1 } - str, err := c.formatBasic(vol) + str, err := c.formatCSIBasic(vol) if err != nil { c.Ui.Error(fmt.Sprintf("Error formatting volume: %s", err)) return 1 @@ -65,7 +65,7 @@ func (c *VolumeStatusCommand) csiStatus(client *api.Client, id string) int { return 0 } -func (c *VolumeStatusCommand) listVolumes(client *api.Client) int { +func (c *VolumeStatusCommand) listCSIVolumes(client *api.Client) int { c.csiBanner() vols, _, err := client.CSIVolumes().List(nil) @@ -182,7 +182,7 @@ func csiFormatSortedVolumes(vols []*api.CSIVolumeListStub) (string, error) { return formatList(rows), nil } -func (c *VolumeStatusCommand) formatBasic(vol *api.CSIVolume) (string, error) { +func (c *VolumeStatusCommand) formatCSIBasic(vol *api.CSIVolume) (string, error) { if c.json || len(c.template) > 0 { out, err := Format(c.json, c.template, vol) if err != nil { diff --git a/command/volume_status_test.go b/command/volume_status_csi_test.go similarity index 100% rename from command/volume_status_test.go rename to command/volume_status_csi_test.go diff --git a/command/volume_status_host.go b/command/volume_status_host.go new file mode 100644 index 00000000000..7878afc55ce --- /dev/null +++ b/command/volume_status_host.go @@ -0,0 +1,180 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "sort" + "strings" + + humanize "github.com/dustin/go-humanize" + "github.com/hashicorp/nomad/api" +) + +func (c *VolumeStatusCommand) hostVolumeStatus(client *api.Client, id, nodeID, nodePool string) int { + if id == "" { + return c.listHostVolumes(client, nodeID, nodePool) + } + + if nodeID != "" || nodePool != "" { + c.Ui.Error("-node or -node-pool options can only be used when no ID is provided") + return 1 + } + + // get a host volume that matches the given prefix or a list of all matches + // if an exact match is not found. note we can't use the shared getByPrefix + // helper here because the List API doesn't match the required signature + + volStub, possible, err := c.getByPrefix(client, id) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error listing volumes: %s", err)) + return 1 + } + if len(possible) > 0 { + out, err := c.formatHostVolumes(possible) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error formatting: %s", err)) + return 1 + } + c.Ui.Error(fmt.Sprintf("Prefix matched multiple volumes\n\n%s", out)) + return 1 + } + + vol, _, err := client.HostVolumes().Get(volStub.ID, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying volume: %s", err)) + return 1 + } + + str, err := c.formatHostVolume(vol) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error formatting volume: %s", err)) + return 1 + } + c.Ui.Output(str) + return 0 +} + +func (c *VolumeStatusCommand) listHostVolumes(client *api.Client, nodeID, nodePool string) int { + vols, _, err := client.HostVolumes().List(&api.HostVolumeListRequest{ + NodeID: nodeID, + NodePool: nodePool, + }, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying volumes: %s", err)) + return 1 + } + + str, err := c.formatHostVolumes(vols) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error formatting volumes: %s", err)) + return 1 + } + c.Ui.Output(str) + + return 0 +} + +func (c *VolumeStatusCommand) getByPrefix(client *api.Client, prefix string) (*api.HostVolumeStub, []*api.HostVolumeStub, error) { + vols, _, err := client.HostVolumes().List(nil, &api.QueryOptions{ + Prefix: prefix, + Namespace: c.namespace, + }) + + if err != nil { + return nil, nil, fmt.Errorf("error querying volumes: %s", err) + } + switch len(vols) { + case 0: + return nil, nil, fmt.Errorf("no volumes with prefix or ID %q found", prefix) + case 1: + return vols[0], nil, nil + default: + // search for exact matches to account for multiple exact ID or name + // matches across namespaces + var match *api.HostVolumeStub + exactMatchesCount := 0 + for _, vol := range vols { + if vol.ID == prefix || vol.Name == prefix { + exactMatchesCount++ + match = vol + } + } + if exactMatchesCount == 1 { + return match, nil, nil + } + return nil, vols, nil + } +} + +func (c *VolumeStatusCommand) formatHostVolume(vol *api.HostVolume) (string, error) { + if c.json || len(c.template) > 0 { + out, err := Format(c.json, c.template, vol) + if err != nil { + return "", fmt.Errorf("format error: %v", err) + } + return out, nil + } + + output := []string{ + fmt.Sprintf("ID|%s", vol.ID), + fmt.Sprintf("Name|%s", vol.Name), + fmt.Sprintf("Namespace|%s", vol.Namespace), + fmt.Sprintf("Plugin ID|%s", vol.PluginID), + fmt.Sprintf("Node ID|%s", vol.NodeID), + fmt.Sprintf("Node Pool|%s", vol.NodePool), + fmt.Sprintf("Capacity|%s", humanize.IBytes(uint64(vol.CapacityBytes))), + fmt.Sprintf("State|%s", vol.State), + fmt.Sprintf("Host Path|%s", vol.HostPath), + } + + // Exit early + if c.short { + return formatKV(output), nil + } + + full := []string{formatKV(output)} + + // Format the allocs + banner := c.Colorize().Color("\n[bold]Allocations[reset]") + allocs := formatAllocListStubs(vol.Allocations, c.verbose, c.length) + full = append(full, banner) + full = append(full, allocs) + + return strings.Join(full, "\n"), nil +} + +func (c *VolumeStatusCommand) formatHostVolumes(vols []*api.HostVolumeStub) (string, error) { + // Sort the output by volume ID + sort.Slice(vols, func(i, j int) bool { return vols[i].ID < vols[j].ID }) + + if c.json || len(c.template) > 0 { + out, err := Format(c.json, c.template, vols) + if err != nil { + return "", fmt.Errorf("format error: %v", err) + } + return out, nil + } + + // Truncate the id unless full length is requested + length := shortId + if c.verbose { + length = fullId + } + + rows := make([]string, len(vols)+1) + rows[0] = "ID|Name|Namespace|Plugin ID|Node ID|Node Pool|State" + for i, v := range vols { + rows[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s", + limit(v.ID, length), + v.Name, + v.Namespace, + v.PluginID, + limit(v.NodeID, length), + v.NodePool, + v.State, + ) + } + return formatList(rows), nil +} diff --git a/command/volume_status_host_test.go b/command/volume_status_host_test.go new file mode 100644 index 00000000000..608a7a64d36 --- /dev/null +++ b/command/volume_status_host_test.go @@ -0,0 +1,150 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/shoenig/test/must" +) + +func TestHostVolumeStatusCommand_Args(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VolumeStatusCommand{Meta: Meta{Ui: ui}} + + code := cmd.Run([]string{ + "-type", "host", + "-node", "6063016a-9d4c-11ef-85fc-9be98efe7e76", + "-node-pool", "prod", + "6e3e80f2-9d4c-11ef-97b1-d38cf64416a4", + }) + must.One(t, code) + + out := ui.ErrorWriter.String() + must.StrContains(t, out, "-node or -node-pool options can only be used when no ID is provided") +} + +func TestHostVolumeStatusCommand_List(t *testing.T) { + ci.Parallel(t) + srv, client, url := testServer(t, true, nil) + t.Cleanup(srv.Shutdown) + + waitForNodes(t, client) + + _, err := client.Namespaces().Register(&api.Namespace{Name: "prod"}, nil) + must.NoError(t, err) + + nodes, _, err := client.Nodes().List(nil) + must.NoError(t, err) + must.Len(t, 1, nodes) + nodeID := nodes[0].ID + + ui := cli.NewMockUi() + + vols := []api.NamespacedID{ + {Namespace: "prod", ID: "database"}, + {Namespace: "prod", ID: "certs"}, + {Namespace: "default", ID: "example"}, + } + + for _, vol := range vols { + hclTestFile := fmt.Sprintf(` +namespace = "%s" +name = "%s" +type = "host" +plugin_id = "plugin_id" +node_id = "%s" +node_pool = "default" +`, vol.Namespace, vol.ID, nodeID) + + file, err := os.CreateTemp(t.TempDir(), "volume-test-*.hcl") + must.NoError(t, err) + _, err = file.WriteString(hclTestFile) + must.NoError(t, err) + + args := []string{"-address", url, file.Name()} + cmd := &VolumeCreateCommand{Meta: Meta{Ui: ui}} + code := cmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + + out := ui.OutputWriter.String() + must.StrContains(t, out, "Created host volume") + ui.OutputWriter.Reset() + } + + cmd := &VolumeStatusCommand{Meta: Meta{Ui: ui}} + args := []string{"-address", url, "-type", "host", "-namespace", "prod"} + code := cmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + out := ui.OutputWriter.String() + must.StrContains(t, out, "certs") + must.StrContains(t, out, "database") + must.StrNotContains(t, out, "example") +} + +func TestHostVolumeStatusCommand_Get(t *testing.T) { + ci.Parallel(t) + srv, client, url := testServer(t, true, nil) + t.Cleanup(srv.Shutdown) + + waitForNodes(t, client) + + _, err := client.Namespaces().Register(&api.Namespace{Name: "prod"}, nil) + must.NoError(t, err) + + nodes, _, err := client.Nodes().List(nil) + must.NoError(t, err) + must.Len(t, 1, nodes) + nodeID := nodes[0].ID + + ui := cli.NewMockUi() + + hclTestFile := fmt.Sprintf(` +namespace = "prod" +name = "example" +type = "host" +plugin_id = "plugin_id" +node_id = "%s" +node_pool = "default" +`, nodeID) + + file, err := os.CreateTemp(t.TempDir(), "volume-test-*.hcl") + must.NoError(t, err) + _, err = file.WriteString(hclTestFile) + must.NoError(t, err) + + args := []string{"-address", url, file.Name()} + regCmd := &VolumeRegisterCommand{Meta: Meta{Ui: ui}} + code := regCmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + + out := ui.OutputWriter.String() + must.StrContains(t, out, "Registered host volume") + parts := strings.Split(out, " ") + id := strings.TrimSpace(parts[len(parts)-1]) + + ui.OutputWriter.Reset() + + // missing the namespace + cmd := &VolumeStatusCommand{Meta: Meta{Ui: ui}} + args = []string{"-address", url, "-type", "host", id} + code = cmd.Run(args) + must.Eq(t, 1, code) + must.StrContains(t, ui.ErrorWriter.String(), + "Error listing volumes: no volumes with prefix or ID") + ui.ErrorWriter.Reset() + + args = []string{"-address", url, "-type", "host", "-namespace", "prod", id} + code = cmd.Run(args) + must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) + out = ui.OutputWriter.String() + must.StrContains(t, out, "example") +}