Google Compute Engine (GCE) storage driver

.docs/user-guide/config.md

@@ -589,6 +589,7 @@ remote storage systems. Currently the following storage drivers are supported:
 [EBS](./storage-providers.md#aws-ebs) | ebs, ec2
 [EFS](./storage-providers.md#aws-efs) | efs
 [RBD](./storage-providers.md#ceph-rbd) | rbd
+[GCEPD](./storage-providers.md#gcepd) | gcepd
 ..more coming|
 
 The `libstorage.server.libstorage.storage.driver` property can be used to

.docs/user-guide/storage-providers.md

@@ -707,3 +707,107 @@ libstorage:
   in place, it may be possible for a client to attach a volume that is already
   attached to another node. Mounting and writing to such a volume could lead to
   data corruption.
+
+## GCEPD
+
+The Google Compute Engine Persistent Disk (GCEPD) driver registers a driver
+named `gcepd` with the `libStorage` driver manager and is used to connect and
+mount Google Compute Engine (GCE) persistent disks with GCE machine instances.
+
+
+### Requirements
+
+* GCE account
+* Service account credentials in JSON for GCE project. If not using the Compute
+  Engine default service account, create a new service account with the Service
+  Account Actor role, and create/download a new private key in JSON format. see
+  [creating a service account](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount)
+  for details.
+
+### Configuration
+
+The following is an example with all possible fields configured. For a running
+example see the `Examples` section.
+
+```yaml
+gcepd:
+  keyfile: /etc/gcekey.json
+  zone: us-west1-b
+  defaultDiskType: pd-ssd
+  tag: rexray
+```
+
+#### Configuration Notes
+
+* The `keyfile` parameter is required. It specifies a path on disk to a file
+  containing the JSON-encoded service account credentials. This file can be
+  downloaded from the GCE web portal.
+* The `zone` parameter is optional, and configures the driver to *only* allow
+  access to the given zone. Creating and listing disks from other zones will be
+  denied. If a zone is not specified, the zone from the client Instance ID will
+  be used when creating new disks.
+* The `defaultDiskType` parameter is optional, and specified what type of disk
+  to create, either `pd-standard` or `pd-ssd`. When not specified, the default
+  is `pd-ssd`.
+* The `tag` parameter is optional, and causes the driver to create or return
+  disks that have a matching tag. The tag is implemented by serializing a JSON
+  structure in to the `Description` field of a GCE disk. Use of this parameter
+  is encouraged, as the driver will only return volumes that have been created
+  by the driver, which is most useful to eliminate listing the boot disks of
+  every GCE disk in your project/zone.
+
+### Runtime behavior
+
+* The GCEPD driver enforces the GCE requirements for disk sizing and naming.
+  Disks must be created with a minimum size of 10GB. Disk names must adhere to
+  the regular expression of [a-z]([-a-z0-9]*[a-z0-9])?, which means the first
+  character must be a lowercase letter, and all following characters must be a
+  dash, lowercase letter, or digit, except the last character, which cannot be a
+  dash.
+* If the `zone` parameter is not specified in the driver configuration, and a
+  request is received to list all volumes that does not specify a zone in the
+  InstanceID header, volumes from all zones will be returned.
+* By default, all disks will be created with type `pd-ssd`, which creates an SSD
+  based disk. If you wish to created disks that are not SSD-based, change the
+  default via the driver config, or the type can be changed at creation time by
+  using the `Type` field of the create request.
+
+### Activating the Driver
+
+To activate the GCEPD driver please follow the instructions for
+[activating storage drivers](./config.md#storage-drivers), using `gcepd` as the
+driver name.
+
+### Troubleshooting
+
+* Make sure that the JSON credentials file as specified in the `keyfile`
+ configuration parameter is present and accessible.
+
+### Examples
+
+Below is a full `config.yml` that works with GCE
+
+```yaml
+libstorage:
+  server:
+    services:
+      gcepd:
+        driver: gcepd
+        gcepd:
+          keyfile: /etc/gcekey.json
+	  tag: rexray
+```
+
+### Caveats
+
+* Snapshot and copy functionality is not yet implemented
+* Most GCE instances can have up to 64 TB of total persistent disk space
+  attached. Shared-core machine types or custom machine types with less than
+  3.75 GB of memory are limited to 3 TB of total persistent disk space. Total
+  persistent disk space for an instance includes the size of the root persistent
+  disk. You can attach up to 16 independent persistent disks to most instances,
+  but instances with shared-core machine types or custom machine types with less
+  than 3.75 GB of memory are limited to a maximum of 4 persistent disks,
+  including the root persistent disk. See
+  [GCE Disks](https://cloud.google.com/compute/docs/disks/) docs for more
+  details.

Makefile

@@ -1075,12 +1075,22 @@ test:
 	$(MAKE) -j parallel-test
 
 test-debug:
-	env LIBSTORAGE_DEBUG=true $(MAKE) test
+	LIBSTORAGE_DEBUG=true $(MAKE) test
 
 test-rbd:
 	DRIVERS=rbd $(MAKE) deps
 	DRIVERS=rbd $(MAKE) ./drivers/storage/rbd/tests/rbd.test
 
+test-rbd-clean:
+	DRIVERS=rbd $(MAKE) clean
+
+test-gcepd:
+	DRIVERS=gcepd $(MAKE) deps
+	DRIVERS=gcepd $(MAKE) ./drivers/storage/gcepd/tests/gcepd.test
+
+test-gcepd-clean:
+	DRIVERS=gcepd $(MAKE) clean
+
 clean: $(GO_CLEAN)
 
 clobber: clean $(GO_CLOBBER)

drivers/storage/gcepd/.gitignore

@@ -0,0 +1 @@
+.vagrant

drivers/storage/gcepd/executor/gce_executor.go

@@ -0,0 +1,109 @@
+// +build !libstorage_storage_executor libstorage_storage_executor_gcepd
+
+package executor
+
+import (
+	"io/ioutil"
+	"path"
+	"regexp"
+
+	gofig "github.com/akutz/gofig/types"
+
+	"github.com/codedellemc/libstorage/api/registry"
+	"github.com/codedellemc/libstorage/api/types"
+	"github.com/codedellemc/libstorage/drivers/storage/gcepd"
+	gceUtils "github.com/codedellemc/libstorage/drivers/storage/gcepd/utils"
+)
+
+const (
+	diskIDPath = "/dev/disk/by-id"
+	diskPrefix = "google-"
+)
+
+// driver is the storage executor for the storage driver.
+type driver struct {
+	config gofig.Config
+}
+
+func init() {
+	registry.RegisterStorageExecutor(gcepd.Name, newDriver)
+}
+
+func newDriver() types.StorageExecutor {
+	return &driver{}
+}
+
+func (d *driver) Init(ctx types.Context, config gofig.Config) error {
+	d.config = config
+	return nil
+}
+
+func (d *driver) Name() string {
+	return gcepd.Name
+}
+
+// Supported returns a flag indicating whether or not the platform
+// implementing the executor is valid for the host on which the executor
+// resides.
+func (d *driver) Supported(
+	ctx types.Context,
+	opts types.Store) (bool, error) {
+
+	return gceUtils.IsGCEInstance(ctx)
+}
+
+// InstanceID returns the instance ID from the current instance from metadata
+func (d *driver) InstanceID(
+	ctx types.Context,
+	opts types.Store) (*types.InstanceID, error) {
+
+	return gceUtils.InstanceID(ctx)
+}
+
+// NextDevice returns the next available device.
+func (d *driver) NextDevice(
+	ctx types.Context,
+	opts types.Store) (string, error) {
+
+	return "", types.ErrNotImplemented
+}
+
+// Retrieve device paths currently attached and/or mounted
+func (d *driver) LocalDevices(
+	ctx types.Context,
+	opts *types.LocalDevicesOpts) (*types.LocalDevices, error) {
+
+	files, err := ioutil.ReadDir(diskIDPath)
+	if err != nil {
+		return nil, err
+	}
+
+	persistentDiskRX, err := regexp.Compile(
+		diskPrefix + `(` + gceUtils.DiskNameRX + `)`)
+	if err != nil {
+		return nil, err
+	}
+
+	attachedDisks, err := gceUtils.GetDisks(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	ld := &types.LocalDevices{Driver: d.Name()}
+	devMap := map[string]string{}
+	for _, f := range files {
+		if persistentDiskRX.MatchString(f.Name()) {
+			matches := persistentDiskRX.FindStringSubmatch(f.Name())
+			volID := matches[1]
+			if _, ok := attachedDisks[volID]; ok && volID != "" {
+				devMap[volID] = path.Join(diskIDPath, f.Name())
+			}
+		}
+	}
+
+	if len(devMap) > 0 {
+		ld.DeviceMap = devMap
+	}
+
+	return ld, nil
+}

drivers/storage/gcepd/gcepd.go

@@ -0,0 +1,44 @@
+// +build !libstorage_storage_driver libstorage_storage_driver_gcepd
+
+package gcepd
+
+import (
+	gofigCore "github.com/akutz/gofig"
+	gofig "github.com/akutz/gofig/types"
+)
+
+const (
+	// Name is the provider's name.
+	Name = "gcepd"
+
+	// InstanceIDFieldProjectID is the key to retrieve the ProjectID value
+	// from the InstanceID Field map.
+	InstanceIDFieldProjectID = "projectID"
+
+	// InstanceIDFieldZone is the key to retrieve the zone value from the
+	// InstanceID Field map.
+	InstanceIDFieldZone = "zone"
+
+	// DiskTypeSSD indicates an SSD based disk should be created
+	DiskTypeSSD = "pd-ssd"
+
+	// DiskTypeStandard indicates a standard (non-SSD) disk
+	DiskTypeStandard = "pd-standard"
+
+	// DefaultDiskType indicates what type of disk to create by default
+	DefaultDiskType = DiskTypeSSD
+)
+
+func init() {
+	r := gofigCore.NewRegistration("GCE")
+	r.Key(gofig.String, "", "",
+		"Required: JSON keyfile for service account", "gcepd.keyfile")
+	r.Key(gofig.String, "", "",
+		"If defined, limit GCE access to given zone", "gcepd.zone")
+	r.Key(gofig.String, "", DefaultDiskType, "Default GCE disk type",
+		"gcepd.defaultDiskType")
+	r.Key(gofig.String, "", "", "Tag to apply and filter disks",
+		"gcepd.tag")
+
+	gofigCore.Register(r)
+}