diff --git a/Makefile b/Makefile index 6de1d1fc..5b8f2b42 100644 --- a/Makefile +++ b/Makefile @@ -1091,6 +1091,13 @@ test-gcepd: test-gcepd-clean: DRIVERS=gcepd $(MAKE) clean +test-digitalocean: + DRIVERS=digitalocean $(MAKE) deps + DRIVERS=digitalocean $(MAKE) ./drivers/storage/digitalocean/tests/digitalocean.test + +test-digitalocean-clean: + DRIVERS=digitalocean $(MAKE) clean + clean: $(GO_CLEAN) clobber: clean $(GO_CLOBBER) diff --git a/drivers/storage/digitalocean/digitalocean.go b/drivers/storage/digitalocean/digitalocean.go new file mode 100644 index 00000000..2e15f1ef --- /dev/null +++ b/drivers/storage/digitalocean/digitalocean.go @@ -0,0 +1,47 @@ +// +build !libstorage_storage_driver libstorage_storage_driver_digitalocean + +package digitalocean + +import ( + gofigCore "github.com/akutz/gofig" + gofig "github.com/akutz/gofig/types" +) + +const ( + // Name is the name of the driver + Name = "digitalocean" + + // InstanceIDFieldRegion is the key used to retrive the region from the + // instance id map + InstanceIDFieldRegion = "region" + + // InstanceIDFieldName is the key used to retrive the name from the instance + // id map + InstanceIDFieldName = "name" + + // VolumePrefix is the value that every DO volume appears with DigitalOcean + // volumes are are found using disk/by-id, ex: + // /dev/disk/by-id/scsi-0DO_Volume_volume-nyc1-01 See + // https://www.digitalocean.com/community/tutorials/how-to-use-block-storage-on-digitalocean#preparing-volumes-for-use-in-linux + VolumePrefix = "scsi-0DO_Volume_" + + // ConfigDOToken is the key for the token in the config file + ConfigDOToken = Name + ".token" + + // ConfigDORegion is the key for the region in the config file + ConfigDORegion = Name + ".region" +) + +func init() { + registerConfig() +} + +func registerConfig() { + r := gofigCore.NewRegistration("DigitalOcean") + r.Key(gofig.String, "", "", "", + ConfigDOToken, + "digitaloceanAccessToken", + "DIGITALOCEAN_ACCESS_TOKEN") + r.Key(gofig.String, "", "", "DigitalOcean region", ConfigDORegion) + gofigCore.Register(r) +} diff --git a/drivers/storage/digitalocean/executor/digitalocean_executor.go b/drivers/storage/digitalocean/executor/digitalocean_executor.go new file mode 100644 index 00000000..5c772a65 --- /dev/null +++ b/drivers/storage/digitalocean/executor/digitalocean_executor.go @@ -0,0 +1,92 @@ +// +build !libstorage_storage_executor libstorage_storage_executor_digitalocean + +package executor + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "regexp" + + gofig "github.com/akutz/gofig/types" + "github.com/codedellemc/libstorage/api/registry" + "github.com/codedellemc/libstorage/api/types" + do "github.com/codedellemc/libstorage/drivers/storage/digitalocean" + doUtils "github.com/codedellemc/libstorage/drivers/storage/digitalocean/utils" +) + +var ( + diskPrefix = regexp.MustCompile(`^` + do.VolumePrefix + `(.*)`) + diskSuffix = regexp.MustCompile("part-.*$") +) + +type driver struct { + config gofig.Config +} + +func init() { + registry.RegisterStorageExecutor(do.Name, newDriver) +} + +func newDriver() types.StorageExecutor { + return &driver{} +} + +func (d *driver) Name() string { + return do.Name +} + +func (d *driver) Init(ctx types.Context, config gofig.Config) error { + d.config = config + return nil +} + +func (d *driver) InstanceID( + ctx types.Context, opts types.Store) (*types.InstanceID, error) { + return doUtils.InstanceID(ctx) +} + +func (d *driver) NextDevice( + ctx types.Context, opts types.Store) (string, error) { + return "", types.ErrNotImplemented +} + +func (d *driver) LocalDevices( + ctx types.Context, + opts *types.LocalDevicesOpts) (*types.LocalDevices, error) { + deviceMap := map[string]string{} + diskIDPath := "/dev/disk/by-id" + + dir, _ := ioutil.ReadDir(diskIDPath) + for _, device := range dir { + switch { + case !diskPrefix.MatchString(device.Name()): + continue + case diskSuffix.MatchString(device.Name()): + continue + case diskPrefix.MatchString(device.Name()): + volumeName := diskPrefix.FindStringSubmatch(device.Name())[1] + devPath, err := filepath.EvalSymlinks( + fmt.Sprintf("%s/%s", diskIDPath, device.Name())) + if err != nil { + return nil, err + } + deviceMap[volumeName] = devPath + } + } + + ld := &types.LocalDevices{Driver: d.Name()} + if len(deviceMap) > 0 { + ld.DeviceMap = deviceMap + } + + return ld, nil +} + +func (d *driver) Supported(ctx types.Context, opts types.Store) (bool, error) { + return doUtils.IsDroplet(ctx) +} + +func (d *driver) token() string { + return d.config.GetString(do.ConfigDOToken) +} diff --git a/drivers/storage/digitalocean/storage/digitalocean_storage.go b/drivers/storage/digitalocean/storage/digitalocean_storage.go new file mode 100644 index 00000000..ef2d620d --- /dev/null +++ b/drivers/storage/digitalocean/storage/digitalocean_storage.go @@ -0,0 +1,362 @@ +// +build !libstorage_storage_driver libstorage_storage_driver_digitalocean + +package storage + +import ( + "fmt" + "path/filepath" + "strconv" + "time" + + log "github.com/Sirupsen/logrus" + gofig "github.com/akutz/gofig/types" + "github.com/akutz/goof" + + "github.com/digitalocean/godo" + + "github.com/codedellemc/libstorage/api/context" + "github.com/codedellemc/libstorage/api/registry" + "github.com/codedellemc/libstorage/api/types" + + do "github.com/codedellemc/libstorage/drivers/storage/digitalocean" + doUtils "github.com/codedellemc/libstorage/drivers/storage/digitalocean/utils" +) + +type driver struct { + name string + config gofig.Config + client *godo.Client +} + +func init() { + registry.RegisterStorageDriver(do.Name, newDriver) +} + +func newDriver() types.StorageDriver { + return &driver{name: do.Name} +} + +func (d *driver) Name() string { + return do.Name +} + +func (d *driver) Init(ctx types.Context, config gofig.Config) error { + d.config = config + token := d.config.GetString(do.ConfigDOToken) + + fields := log.Fields{ + "token": token, + } + + if token == "" { + fields["token"] = "" + } else { + fields["token"] = "******" + } + + fields["region"] = d.config.GetString(do.ConfigDORegion) + + client, err := doUtils.Client(token) + if err != nil { + return err + } + d.client = client + + log.WithFields(fields).Info("storage driver initialized") + + return nil +} + +func (d *driver) Type(ctx types.Context) (types.StorageType, error) { + return types.Block, nil +} + +// DigitalOcean volumes are are found using device-by-id, ex: +// /dev/disk/by-id/scsi-0DO_Volume_volume-nyc1-01 See +// https://www.digitalocean.com/community/tutorials/how-to-use-block-storage-on-digitalocean#preparing-volumes-for-use-in-linux +func (d *driver) NextDeviceInfo( + ctx types.Context) (*types.NextDeviceInfo, error) { + return nil, nil +} + +func (d *driver) InstanceInspect( + ctx types.Context, opts types.Store) (*types.Instance, error) { + iid := context.MustInstanceID(ctx) + return &types.Instance{ + InstanceID: iid, + Region: iid.Fields[do.InstanceIDFieldRegion], + Name: iid.Fields[do.InstanceIDFieldName], + ProviderName: iid.Driver, + }, nil +} + +func (d *driver) Volumes( + ctx types.Context, opts *types.VolumesOpts) ([]*types.Volume, error) { + doVolumes, _, err := d.client.Storage.ListVolumes(nil) + if err != nil { + return nil, err + } + + var volumes []*types.Volume + for _, vol := range doVolumes { + volumes = append(volumes, d.toTypesVolume(ctx, &vol, opts.Attachments)) + } + + return volumes, nil +} + +func (d *driver) VolumeInspect( + ctx types.Context, volumeID string, opts *types.VolumeInspectOpts) (*types.Volume, error) { + doVolume, _, err := d.client.Storage.GetVolume(volumeID) + if err != nil { + return nil, err + } + + volume := d.toTypesVolume(ctx, doVolume, opts.Attachments) + return volume, nil +} + +func (d *driver) VolumeCreate( + ctx types.Context, name string, opts *types.VolumeCreateOpts) (*types.Volume, error) { + if opts.AvailabilityZone == nil { + instance, err := d.InstanceInspect(ctx, nil) + if err != nil { + return nil, err + } + opts.AvailabilityZone = &instance.Region + } + volumeReq := &godo.VolumeCreateRequest{ + Region: *opts.AvailabilityZone, + Name: name, + SizeGigaBytes: *opts.Size, + } + + volume, _, err := d.client.Storage.CreateVolume(volumeReq) + if err != nil { + return nil, err + } + + return d.VolumeInspect(ctx, volume.ID, &types.VolumeInspectOpts{ + Attachments: types.VolAttReqTrue, + }) +} + +func (d *driver) VolumeCreateFromSnapshot( + ctx types.Context, snapshotID string, volumeName string, + opts *types.VolumeCreateOpts) (*types.Volume, error) { + return nil, types.ErrNotImplemented +} + +func (d *driver) VolumeCopy( + ctx types.Context, volumeID string, volumeName string, + opts types.Store) (*types.Volume, error) { + return nil, types.ErrNotImplemented +} + +func (d *driver) VolumeSnapshot( + ctx types.Context, volumeID string, snapshotName string, + opts types.Store) (*types.Snapshot, error) { + return nil, types.ErrNotImplemented +} + +func (d *driver) VolumeRemove( + ctx types.Context, + volumeID string, + opts *types.VolumeRemoveOpts) error { + volume, _, err := d.client.Storage.GetVolume(volumeID) + if err != nil { + return err + } + + if len(volume.DropletIDs) > 0 { + if !opts.Force { + return goof.New("volume already attached") + } + + err := d.volumeDetach(volumeID) + if err != nil { + return err + } + } + + _, err = d.client.Storage.DeleteVolume(volumeID) + if err != nil { + return err + } + + return nil +} + +func (d *driver) volumeDetach(volumeID string) error { + action, _, err := d.client.StorageActions.Detach(volumeID) + if err != nil { + return err + } + + err = d.waitForAction(volumeID, action) + if err != nil { + return err + } + return nil +} + +func (d *driver) VolumeAttach( + ctx types.Context, volumeID string, + opts *types.VolumeAttachOpts) (*types.Volume, string, error) { + vol, _, err := d.client.Storage.GetVolume(volumeID) + if err != nil { + return nil, "", goof.WithError("error retrieving volume", err) + } + + if len(vol.DropletIDs) > 0 { + if !opts.Force { + return nil, "", goof.New("volume already attached") + } + + err = d.volumeDetach(volumeID) + if err != nil { + return nil, "", err + } + } + + dropletID := mustInstanceIDID(ctx) + dropletIDI, err := strconv.Atoi(*dropletID) + if err != nil { + return nil, "", err + } + + action, _, err := d.client.StorageActions.Attach(volumeID, dropletIDI) + if err != nil { + return nil, "", err + } + + err = d.waitForAction(volumeID, action) + if err != nil { + return nil, "", err + } + + attachedVol, err := d.VolumeInspect(ctx, volumeID, &types.VolumeInspectOpts{ + Attachments: types.VolAttReqTrue}) + if err != nil { + return nil, "", goof.WithError("error retrieving volume", err) + } + + return attachedVol, attachedVol.Attachments[0].DeviceName, nil +} + +func (d *driver) VolumeDetach( + ctx types.Context, volumeID string, + opts *types.VolumeDetachOpts) (*types.Volume, error) { + vol, _, err := d.client.Storage.GetVolume(volumeID) + if err != nil { + return nil, goof.WithError("error getting volume", err) + } + + if len(vol.DropletIDs) == 0 { + return nil, goof.WithError("volume already detached", err) + } + + err = d.volumeDetach(volumeID) + if err != nil { + return nil, err + } + + ctx.Info("detached volume", volumeID) + + detachedVol, err := d.VolumeInspect(ctx, volumeID, &types.VolumeInspectOpts{ + Attachments: types.VolAttReqTrue, + Opts: opts.Opts, + }) + if err != nil { + return nil, goof.WithError("error getting volume information", err) + } + + return detachedVol, nil +} + +func (d *driver) Snapshots( + ctx types.Context, opts types.Store) ([]*types.Snapshot, error) { + return nil, types.ErrNotImplemented +} + +func (d *driver) SnapshotInspect( + ctx types.Context, snapshotID string, + opts types.Store) (*types.Snapshot, error) { + return nil, types.ErrNotImplemented +} + +func (d *driver) SnapshotCopy( + ctx types.Context, snapshotID string, snapshotName string, destinationID string, + opts types.Store) (*types.Snapshot, error) { + return nil, types.ErrNotImplemented +} + +func (d *driver) SnapshotRemove( + ctx types.Context, snapshotID string, opts types.Store) error { + return types.ErrNotImplemented +} + +func mustInstanceIDID(ctx types.Context) *string { + return &context.MustInstanceID(ctx).ID +} + +func (d *driver) toTypesVolume( + ctx types.Context, volume *godo.Volume, + attachments types.VolumeAttachmentsTypes) *types.Volume { + // Collect attachment info for the volume + var atts []*types.VolumeAttachment + if attachments.Requested() { + for _, id := range volume.DropletIDs { + attachment := &types.VolumeAttachment{ + VolumeID: volume.ID, + InstanceID: &types.InstanceID{ + ID: strconv.Itoa(id), + Driver: d.Name(), + }, + } + + if attachments.Devices() { + attachment.DeviceName, _ = filepath.EvalSymlinks( + fmt.Sprintf( + "%s/%s", "/dev/disk/by-id", + do.VolumePrefix+volume.Name)) + } + atts = append(atts, attachment) + } + } + + status := "attached" + if len(atts) < 1 { + status = "detached" + } + + vol := &types.Volume{ + Name: volume.Name, + ID: volume.ID, + Encrypted: false, + Size: volume.SizeGigaBytes, + AvailabilityZone: volume.Region.Slug, + Attachments: atts, + Status: status, + } + + return vol +} + +func (d *driver) waitForAction(volumeID string, action *godo.Action) error { + // TODO expose these ints as options + for i := 0; i < 10; i++ { + duration := i * 15 + time.Sleep(time.Duration(duration) * time.Millisecond) + + action, _, err := d.client.StorageActions.Get(volumeID, action.ID) + if err != nil { + return err + } + if action.Status == godo.ActionCompleted { + break + } + } + + return nil +} diff --git a/drivers/storage/digitalocean/tests/README.md b/drivers/storage/digitalocean/tests/README.md new file mode 100644 index 00000000..d3185bc6 --- /dev/null +++ b/drivers/storage/digitalocean/tests/README.md @@ -0,0 +1,53 @@ +# Testing the DigitalOcean triver + +The tests for the DigitalOcean driver assume that you have access to a +DigitalOcean account that has a token that with read/write access. There are +[docs](https://www.digitalocean.com/community/tutorials/how-to-use-the-digitalocean-api-v2) +on how to do so if you do not already have one. + +# Setting up an environment +The tests require a droplet running in a region that supports storage volumes. +You can find out which regions support volumes using the +[api](https://developers.digitalocean.com/documentation/v2/#regions) directly +or use the scripts included in the repo. + +## Using Terraform +The scripts require that you have +[Terraform](https://github.com/hashicorp/terraform) available locally. You will +also to have access to a ssh key associated with your DigitalOcean account +([directions +here](https://www.digitalocean.com/community/tutorials/how-to-configure-ssh-key-based-authentication-on-a-linux-server#how-to-embed-your-public-key-when-creating-your-server)). +You will need to pass the ssh key id or fingerprint to the setup scripts so +that Terraform can spin up you droplet. + +#### Starting an environment + +``` +cd drivers/storage/digitalocean/tests +./test-env-up $SSH_KEY_ID +``` + +#### Executing the tests +Once you have your environment set up, you can build and copy the tests to your running droplet. + +Running the build comamand: +``` +GOOS=linux GOARCH=amd64 BUILD_TAGS="gofig pflag libstorage_integration_docker libstorage_storage_driver libstorage_storage_executor libstorage_storage_driver_digitalocean libstorage_storage_executor_digitalocean" make build-tests +``` +will create a `digitalocean.test` in the tests directory. You can scp that to +your droplet and then run the tests. You will also need to configure libstorage +to use the digitalocean driver by setting the following fields in +`/etc/libstorage/config.yaml`: + +``` +digitalocean: + token: $YOUR_API_KEY + # You can use other regions here + region: sfo2 +``` + +#### Deleting an environment +``` +cd drivers/storage/digitalocean/tests +./test-env-down $SSH_KEY_ID +``` diff --git a/drivers/storage/digitalocean/tests/coverage.mk b/drivers/storage/digitalocean/tests/coverage.mk new file mode 100644 index 00000000..10df4340 --- /dev/null +++ b/drivers/storage/digitalocean/tests/coverage.mk @@ -0,0 +1,2 @@ +DIGITALOCEAN_COVERPKG := $(ROOT_IMPORT_PATH)/drivers/storage/digitalocean +TEST_COVERPKG_./drivers/storage/digitalocean/tests := $(DIGITALOCEAN_COVERPKG),$(DIGITALOCEAN_COVERPKG)/executor diff --git a/drivers/storage/digitalocean/tests/digitalocean_test.go b/drivers/storage/digitalocean/tests/digitalocean_test.go new file mode 100644 index 00000000..0c335db0 --- /dev/null +++ b/drivers/storage/digitalocean/tests/digitalocean_test.go @@ -0,0 +1,356 @@ +// +build !libstorage_storage_driver libstorage_storage_driver_digitalocean + +package digitalocean + +import ( + "os" + "strconv" + "strings" + "testing" + + log "github.com/Sirupsen/logrus" + gofigCore "github.com/akutz/gofig" + gofig "github.com/akutz/gofig/types" + "github.com/codedellemc/libstorage/api/context" + "github.com/codedellemc/libstorage/api/registry" + "github.com/codedellemc/libstorage/api/server" + apitests "github.com/codedellemc/libstorage/api/tests" + "github.com/codedellemc/libstorage/api/types" + "github.com/codedellemc/libstorage/api/utils" + do "github.com/codedellemc/libstorage/drivers/storage/digitalocean" + doUtils "github.com/codedellemc/libstorage/drivers/storage/digitalocean/utils" + "github.com/stretchr/testify/assert" +) + +var ( + configYAML = []byte(` +digitalocean: + token: 12345 + region: sfo2`) +) + +func skipTests() bool { + travis, _ := strconv.ParseBool(os.Getenv("TRAVIS")) + noTest, _ := strconv.ParseBool(os.Getenv("TEST_SKIP_DO")) + return travis || noTest +} + +var volumeName string +var volumeName2 string + +func init() { + uuid, _ := types.NewUUID() + uuids := strings.Split(uuid.String(), "-") + volumeName = uuids[0] + if _, err := strconv.Atoi(string(volumeName[0])); err == nil { + // TODO randomly select a-z here + volumeName = strings.Join([]string{"a", volumeName[1:]}, "") + } + + uuid, _ = types.NewUUID() + uuids = strings.Split(uuid.String(), "-") + volumeName2 = uuids[0] + if _, err := strconv.Atoi(string(volumeName2[0])); err == nil { + // TODO randomly select a-z here + volumeName2 = strings.Join([]string{"a", volumeName2[1:]}, "") + } +} + +func TestMain(m *testing.M) { + server.CloseOnAbort() + ec := m.Run() + os.Exit(ec) +} + +func TestConfig(t *testing.T) { + if skipTests() { + t.SkipNow() + } + + tfDO := func(config gofig.Config, client types.Client, t *testing.T) { + assert.NotEqual(t, config.GetString(do.ConfigDORegion), "") + assert.NotEqual(t, config.GetString(do.ConfigDOToken), "") + } + + apitests.Run(t, do.Name, configYAML, tfDO) +} + +func TestInstanceID(t *testing.T) { + if skipTests() { + t.SkipNow() + } + + sd, err := registry.NewStorageDriver(do.Name) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + if err := sd.Init(ctx, gofigCore.New()); err != nil { + t.Fatal(err) + } + + iid, err := doUtils.InstanceID(ctx) + if err != nil { + t.Fatal(err.Error()) + } + + ctx = ctx.WithValue(context.InstanceIDKey, iid) + i, err := sd.InstanceInspect(ctx, utils.NewStore()) + if err != nil { + t.Fatal(err) + } + + iid = i.InstanceID + apitests.Run( + t, do.Name, nil, + (&apitests.InstanceIDTest{ + Driver: do.Name, + Expected: iid, + }).Test) +} + +func TestServices(t *testing.T) { + if skipTests() { + t.SkipNow() + } + + tf := func(config gofig.Config, client types.Client, t *testing.T) { + reply, err := client.API().Services(nil) + assert.NoError(t, err) + assert.Equal(t, len(reply), 1) + + _, ok := reply[do.Name] + assert.True(t, ok) + } + + apitests.Run(t, do.Name, configYAML, tf) +} + +func TestVolumeAttach(t *testing.T) { + if skipTests() { + t.SkipNow() + } + var vol *types.Volume + tf := func(config gofig.Config, client types.Client, t *testing.T) { + vol = volumeCreate(t, client, volumeName) + _ = volumeAttach(t, client, vol.ID) + _ = volumeInspectAttached(t, client, vol.ID) + _ = volumeInspectDetachedFail(t, client, vol.ID) + _ = volumeDetach(t, client, vol.ID) + _ = volumeInspectDetached(t, client, vol.ID) + volumeRemove(t, client, vol.ID) + } + apitests.Run(t, do.Name, configYAML, tf) +} + +func TestVolumeCreateRemove(t *testing.T) { + if skipTests() { + t.SkipNow() + } + + tf := func(config gofig.Config, client types.Client, t *testing.T) { + vol := volumeCreate(t, client, volumeName) + volumeRemove(t, client, vol.ID) + } + + apitests.Run(t, do.Name, configYAML, tf) +} + +func TestVolumes(t *testing.T) { + if skipTests() { + t.SkipNow() + } + + tf := func(config gofig.Config, client types.Client, t *testing.T) { + _ = volumeCreate(t, client, volumeName) + _ = volumeCreate(t, client, volumeName2) + + vol1 := volumeByName(t, client, volumeName) + vol2 := volumeByName(t, client, volumeName2) + + volumeRemove(t, client, vol1.ID) + volumeRemove(t, client, vol2.ID) + } + + apitests.Run(t, do.Name, configYAML, tf) +} + +// Test implementation functions + +func volumeCreate(t *testing.T, client types.Client, + volumeName string) *types.Volume { + log.WithField("volumeName", volumeName).Info("creating volume") + + size := int64(10) + opts := map[string]interface{}{ + "priority": 2, + "owner": "libstorage@example.com", + } + + volumeCreateRequest := &types.VolumeCreateRequest{ + Name: volumeName, + Size: &size, + Opts: opts, + } + + // Send request and retrieve created libStorage types.Volume + reply, err := client.API().VolumeCreate( + nil, do.Name, volumeCreateRequest) + assert.NoError(t, err) + if err != nil { + t.FailNow() + t.Error("failed volumeCreate") + } + apitests.LogAsJSON(reply, t) + + // Check if name and size are same + assert.Equal(t, volumeName, reply.Name) + assert.Equal(t, size, reply.Size) + return reply +} + +func volumeByName( + t *testing.T, client types.Client, volumeName string) *types.Volume { + log.WithField("volumeName", volumeName).Info( + "get volume by digitalocean.Name") + vols, err := client.API().Volumes(nil, 0) + assert.NoError(t, err) + if err != nil { + t.FailNow() + } + + assert.Contains(t, vols, do.Name) + for _, vol := range vols[do.Name] { + if vol.Name == volumeName { + return vol + } + } + // No matching volumes found + t.FailNow() + t.Error("failed volumeByName") + return nil +} + +func volumeByID( + t *testing.T, client types.Client, volumeID string) *types.Volume { + log.WithField("volumeID", volumeID).Info( + "get volume by digitalocean.Name using ID") + // Retrieve all volumes + vols, err := client.API().Volumes(nil, 0) + assert.NoError(t, err) + if err != nil { + t.FailNow() + } + // Filter volumes to those under the digitalocean service, + // and find a volume matching inputted volume ID + assert.Contains(t, vols, do.Name) + for _, vol := range vols[do.Name] { + if vol.ID == volumeID { + return vol + } + } + // No matching volumes found + t.FailNow() + t.Error("failed volumeByID") + return nil +} + +func volumeRemove(t *testing.T, client types.Client, volumeID string) { + log.WithField("volumeID", volumeID).Info("removing volume") + err := client.API().VolumeRemove(nil, do.Name, volumeID, false) + + assert.NoError(t, err) + if err != nil { + t.Error("failed volumeRemove") + t.FailNow() + } +} + +func volumeAttach( + t *testing.T, client types.Client, volumeID string) *types.Volume { + log.WithField("volumeID", volumeID).Info("attaching volume") + + reply, token, err := client.API().VolumeAttach( + nil, do.Name, volumeID, &types.VolumeAttachRequest{}) + + assert.NoError(t, err) + if err != nil { + t.Error("failed volumeAttach") + t.FailNow() + } + apitests.LogAsJSON(reply, t) + assert.NotEqual(t, token, "") + + return reply +} + +func volumeInspectAttached( + t *testing.T, client types.Client, volumeID string) *types.Volume { + log.WithField("volumeID", volumeID).Info("inspecting volume") + + reply, err := client.API().VolumeInspect( + nil, do.Name, volumeID, + types.VolAttReqTrue) + assert.NoError(t, err) + + if err != nil { + t.Error("failed volumeInspectAttached") + t.FailNow() + } + apitests.LogAsJSON(reply, t) + assert.Len(t, reply.Attachments, 1) + return reply +} + +func volumeInspectDetached( + t *testing.T, client types.Client, volumeID string) *types.Volume { + log.WithField("volumeID", volumeID).Info("inspecting volume") + + reply, err := client.API().VolumeInspect( + nil, do.Name, volumeID, + types.VolAttReq) + + assert.NoError(t, err) + if err != nil { + t.Error("failed volumeInspectDetached") + t.FailNow() + } + + apitests.LogAsJSON(reply, t) + assert.Len(t, reply.Attachments, 0) + return reply +} + +func volumeInspectDetachedFail( + t *testing.T, client types.Client, volumeID string) *types.Volume { + + log.WithField("volumeID", volumeID).Info("inspecting volume") + reply, err := client.API().VolumeInspect(nil, do.Name, volumeID, 0) + assert.NoError(t, err) + + if err != nil { + t.Error("failed volumeInspectDetachedFail") + t.FailNow() + } + apitests.LogAsJSON(reply, t) + assert.Len(t, reply.Attachments, 0) + return reply +} + +func volumeDetach( + t *testing.T, client types.Client, volumeID string) *types.Volume { + log.WithField("volumeID", volumeID).Info("detaching volume") + + reply, err := client.API().VolumeDetach( + nil, do.Name, volumeID, &types.VolumeDetachRequest{}) + + assert.NoError(t, err) + if err != nil { + t.Error("failed volumeDetach") + t.FailNow() + } + apitests.LogAsJSON(reply, t) + assert.Len(t, reply.Attachments, 0) + return reply +} diff --git a/drivers/storage/digitalocean/tests/terraform/.gitignore b/drivers/storage/digitalocean/tests/terraform/.gitignore new file mode 100644 index 00000000..49d1ef28 --- /dev/null +++ b/drivers/storage/digitalocean/tests/terraform/.gitignore @@ -0,0 +1 @@ +terraform.tfstate* diff --git a/drivers/storage/digitalocean/tests/terraform/main.tf b/drivers/storage/digitalocean/tests/terraform/main.tf new file mode 100644 index 00000000..b7cb5757 --- /dev/null +++ b/drivers/storage/digitalocean/tests/terraform/main.tf @@ -0,0 +1,25 @@ +resource digitalocean_volume "volume" { + region = "${var.region}" + size = "${var.volume_size}" + name = "libstorage-volume" +} + +data "template_file" "cloudinit" { + template = "${file("${path.module}/templates/cloudinit.tpl")}" + + vars = { + region = "${var.region}" + } +} + +resource digitalocean_droplet "droplet" { + region = "${var.region}" + size = "${var.size}" + image = "${var.image}" + name = "libstorage-integration" + + user_data = "${data.template_file.cloudinit.rendered}" + + ssh_keys = ["${var.ssh_key}"] + volume_ids = ["${digitalocean_volume.volume.id}"] +} diff --git a/drivers/storage/digitalocean/tests/terraform/templates/cloudinit.tpl b/drivers/storage/digitalocean/tests/terraform/templates/cloudinit.tpl new file mode 100644 index 00000000..787badec --- /dev/null +++ b/drivers/storage/digitalocean/tests/terraform/templates/cloudinit.tpl @@ -0,0 +1,7 @@ +#cloud-config +write_files: + - path: /home/core/.bashrc + permissions: 0644 + owner: 'core:core' + content: | + export DIGITALOCEAN_REGION=${region} diff --git a/drivers/storage/digitalocean/tests/terraform/variables.tf b/drivers/storage/digitalocean/tests/terraform/variables.tf new file mode 100644 index 00000000..1a7fdde2 --- /dev/null +++ b/drivers/storage/digitalocean/tests/terraform/variables.tf @@ -0,0 +1,28 @@ +variable region { + description = </dev/null || { + echo >&2 "You need to install terraform: https://www.terraform.io/downloads.html" + exit 1 +} + +SSH_KEY_ID=$1 + +usage() { + echo "Usage: ${0}" ssh-key-id + echo "" + echo "Requires that the DIGITALOCEAN_ACCESS_TOKEN environment variable is set\nA new server will be started in sfo2" +} + +if [ -z "$DIGITALOCEAN_TOKEN" ] || [ -z "$SSH_KEY_ID" ]; then + usage + exit 1 +fi + +cd terraform && terraform destroy -force -var "ssh_key=$SSH_KEY_ID" diff --git a/drivers/storage/digitalocean/tests/test-env-up.sh b/drivers/storage/digitalocean/tests/test-env-up.sh new file mode 100755 index 00000000..87e23ce5 --- /dev/null +++ b/drivers/storage/digitalocean/tests/test-env-up.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e + +hash terraform 2>/dev/null || { + echo >&2 "You need to install terraform: https://www.terraform.io/downloads.html" + exit 1 +} + +SSH_KEY_ID=$1 + +usage() { + echo "Usage: ${0}" ssh-key-id + echo "" + echo "Requires that the DIGITALOCEAN_TOKEN environment variable is set\nA new server will be started in sfo2" +} + +if [ -z "$DIGITALOCEAN_TOKEN" ] || [ -z "$SSH_KEY_ID" ]; then + usage + exit 1 +fi + +cd terraform && terraform apply -var "ssh_key=$SSH_KEY_ID" diff --git a/drivers/storage/digitalocean/utils/client.go b/drivers/storage/digitalocean/utils/client.go new file mode 100644 index 00000000..cdd2a6da --- /dev/null +++ b/drivers/storage/digitalocean/utils/client.go @@ -0,0 +1,24 @@ +// +build !libstorage_storage_driver libstorage_storage_driver_digitalocean + +package utils + +import ( + "github.com/codedellemc/libstorage/api" + "github.com/digitalocean/godo" + "golang.org/x/oauth2" +) + +// Client returns a new DigitalOcean client +func Client(token string) (*godo.Client, error) { + tokenSrc := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + + client, err := godo.New(oauth2.NewClient( + oauth2.NoContext, tokenSrc), + godo.SetUserAgent(userAgent())) + + return client, err +} + +func userAgent() string { + return "libstorage/" + api.Version.SemVer +} diff --git a/drivers/storage/digitalocean/utils/instance.go b/drivers/storage/digitalocean/utils/instance.go new file mode 100644 index 00000000..562f6d30 --- /dev/null +++ b/drivers/storage/digitalocean/utils/instance.go @@ -0,0 +1,75 @@ +// +build !libstorage_storage_driver libstorage_storage_driver_digitalocean + +package utils + +import ( + "io/ioutil" + "net/http" + + "github.com/codedellemc/libstorage/api/types" + "github.com/codedellemc/libstorage/drivers/storage/digitalocean" +) + +const ( + metadataBase = "169.254.169.254" + metadataURL = "http://" + metadataBase + "/metadata/v1" + metadataID = metadataURL + "/id" + metadataRegion = metadataURL + "/region" + metadataName = metadataURL + "/hostname" +) + +// InstanceID gets the instance information from the droplet +func InstanceID(ctx types.Context) (*types.InstanceID, error) { + + id, err := getURL(ctx, metadataID) + if err != nil { + return nil, err + } + + region, err := getURL(ctx, metadataRegion) + if err != nil { + return nil, err + } + + name, err := getURL(ctx, metadataName) + if err != nil { + return nil, err + } + + return &types.InstanceID{ + ID: id, + Driver: digitalocean.Name, + Fields: map[string]string{ + digitalocean.InstanceIDFieldRegion: region, + digitalocean.InstanceIDFieldName: name, + }, + }, nil +} + +// IsDroplet is a simple check to see if code is being executed on a DigitalOcean droplet or not +func IsDroplet(ctx types.Context) (bool, error) { + _, err := getURL(ctx, metadataURL) + if err != nil { + return false, nil + } + return true, nil +} + +func getURL(ctx types.Context, url string) (string, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + + resp, err := doRequest(ctx, req) + if err != nil { + return "", err + } + defer resp.Body.Close() + id, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(id), nil +} diff --git a/drivers/storage/digitalocean/utils/utils_go1.7.go b/drivers/storage/digitalocean/utils/utils_go1.7.go new file mode 100644 index 00000000..c536148d --- /dev/null +++ b/drivers/storage/digitalocean/utils/utils_go1.7.go @@ -0,0 +1,22 @@ +// +build go1.7 +// +build !libstorage_storage_driver libstorage_storage_driver_digitalocean + +package utils + +import ( + "net/http" + + "github.com/codedellemc/libstorage/api/types" +) + +func doRequest(ctx types.Context, req *http.Request) (*http.Response, error) { + return doRequestWithClient(ctx, http.DefaultClient, req) +} + +func doRequestWithClient( + ctx types.Context, + client *http.Client, + req *http.Request) (*http.Response, error) { + req = req.WithContext(ctx) + return client.Do(req) +} diff --git a/drivers/storage/digitalocean/utils/utils_pre_go17.go b/drivers/storage/digitalocean/utils/utils_pre_go17.go new file mode 100644 index 00000000..0c9abfe8 --- /dev/null +++ b/drivers/storage/digitalocean/utils/utils_pre_go17.go @@ -0,0 +1,23 @@ +// +build !go1.7 +// +build !libstorage_storage_driver libstorage_storage_driver_digitalocean + +package utils + +import ( + "net/http" + + "golang.org/x/net/context/ctxhttp" + + "github.com/codedellemc/libstorage/api/types" +) + +func doRequest(ctx types.Context, req *http.Request) (*http.Response, error) { + return doRequestWithClient(ctx, http.DefaultClient, req) +} + +func doRequestWithClient( + ctx types.Context, + client *http.Client, + req *http.Request) (*http.Response, error) { + return ctxhttp.Do(ctx, client, req) +} diff --git a/glide.lock b/glide.lock index 061d11d2..4671509c 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,8 @@ -hash: 626a0c16518d722d7925eeb2e610cd5f565c2160072414a99b4e1067aac5685a -updated: 2017-02-13T10:56:51.332372709-07:00 +hash: ba391adbccda43b43812c633d8423a72e769887a4561c0f250288b4c5fa6b8ac +updated: 2017-02-14T21:13:06.854781561-08:00 imports: - name: cloud.google.com/go - version: b4ca3d4ba32e251f6fee7bda65c5727ccbf3faa9 + version: e4de3dc4493f142c5833f3185e1182025a61f805 subpackages: - compute/metadata - internal @@ -23,7 +23,7 @@ imports: - vboxwebsrv - virtualboxclient - name: github.com/asaskevich/govalidator - version: 7b3beb6df3c42abd3509abfc3bcacc0fbfb7c877 + version: fdf19785fd3558d619ef81212f5edf1d6c2a5911 - name: github.com/aws/aws-sdk-go version: 3f8f870ec9939e32b3372abf74d24e468bcd285d repo: https://github.com/aws/aws-sdk-go @@ -83,25 +83,31 @@ imports: subpackages: - logrus - name: github.com/davecgh/go-spew - version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 + version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 subpackages: - spew +- name: github.com/digitalocean/godo + version: 2ff8a02a86cd6918b384a5000ceebe886844fbce - name: github.com/fsnotify/fsnotify - version: fd9ec7deca8bf46ecd2a795baaacf2b3a9be1197 + version: a904159b9206978bb6d53fcc7a769e5cd726c737 - name: github.com/go-ini/ini - version: 6e4869b434bd001f6983749881c7ead3545887d8 + version: ee900ca565931451fe4e4409bcbd4316331cec1c - name: github.com/golang/protobuf version: 8ee79997227bf9b34611aee7946ae64735e6fd93 subpackages: - proto +- name: github.com/google/go-querystring + version: 53e6ce116135b80d037921a7fdd5138cf32d7a8a + subpackages: + - query - name: github.com/googleapis/gax-go version: da06d194a00e19ce00d9011a13931c3f6f6887c7 - name: github.com/gorilla/context version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 - name: github.com/gorilla/mux - version: 757bef944d0f21880861c2dd9c871ca543023cba + version: 392c28fe23e1c45ddba891b0320b3b5df220beea - name: github.com/hashicorp/hcl - version: f74cf8281543a0797d7b4ab7d88e76e7ba125308 + version: 372e8ddaa16fd67e371e9323807d056b799360af subpackages: - hcl/ast - hcl/parser @@ -120,20 +126,14 @@ imports: version: 9b883c5eb462dd5cb1b0a7a104fe86bc6b9bd391 repo: https://github.com/kardianos/osext.git vcs: git -- name: github.com/kr/fs - version: 2788f0dbd16903de03cb8186e5c7d97b69ad387b - name: github.com/magiconair/properties - version: 0723e352fa358f9322c938cc2dadda874e9151a9 + version: b3b15ef068fd0b17ddf408a23669f20811d194d2 - name: github.com/mitchellh/mapstructure - version: f3009df150dadf309fdee4a54ed65c124afad715 + version: db1efb556f84b25a0a13a04aad883943538ad2e0 - name: github.com/pelletier/go-buffruneio version: df1e16fde7fc330a0ca68167c23bf7ed6ac31d6d - name: github.com/pelletier/go-toml - version: 45932ad32dfdd20826f5671da37a5f3ce9f26a8d -- name: github.com/pkg/errors - version: 248dadf4e9068a0b3e79f02ed0a610d935de5302 -- name: github.com/pkg/sftp - version: 4d0e916071f68db74f8a73926335f809396d6b42 + version: c9506ee96398e7571356462217b9e24d6a628d71 - name: github.com/pmezard/go-difflib version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: @@ -157,29 +157,23 @@ imports: version: 5f376aa629ac60c3215cc368e674bd996093a01a repo: https://github.com/akutz/logrus - name: github.com/spf13/afero - version: 52e4a6cfac46163658bd4f123c49b6ee7dc75f78 + version: 72b31426848c6ef12a7a8e216708cb0d1530f074 subpackages: - mem - - sftp - name: github.com/spf13/cast - version: 2580bc98dc0e62908119e4737030cc2fdfc45e4c + version: d1139bab1c07d5ad390a65e7305876b3c1a8370b - name: github.com/spf13/jwalterweatherman - version: 33c24e77fb80341fe7130ee7c594256ff08ccc46 + version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 - name: github.com/spf13/pflag version: 5ccb023bc27df288a957c5e994cd44fd19619465 - name: github.com/spf13/viper version: 651d9d916abc3c3d6a91a12549495caba5edffd2 - name: github.com/stretchr/testify - version: 976c720a22c8eb4eb6a0b4348ad85ad12491a506 + version: 4d4bfba8f1d1027c4fdbe371823030df51419987 subpackages: - assert -- name: golang.org/x/crypto - version: 9477e0b78b9ac3d0b03822fd95422e2fe07627cd - subpackages: - - curve25519 - - ed25519 - - ed25519/internal/edwards25519 - - ssh +- name: github.com/tent/http-link-go + version: ac974c61c2f990f4115b119354b5e0b47550e888 - name: golang.org/x/net version: b336a971b799939dd16ae9b1df8334cb8b977c4d subpackages: @@ -203,7 +197,7 @@ imports: subpackages: - unix - name: golang.org/x/text - version: a8b38433e35b65ba247bb267317037dee1b70cea + version: 06d6eba81293389cafdff7fca90d75592194b2d9 subpackages: - transform - unicode/norm @@ -216,7 +210,7 @@ imports: - googleapi - googleapi/internal/uritemplates - name: google.golang.org/appengine - version: 8758a385849434ba5eac8aeedcf5192c5a0f5f10 + version: 2e4a801b39fc199db615bfca7d0b9f8cd9580599 subpackages: - internal - internal/app_identity @@ -228,7 +222,7 @@ imports: - internal/urlfetch - urlfetch - name: google.golang.org/grpc - version: 50955793b0183f9de69bd78e2ec251cf20aab121 + version: cbcceb2942a489498cf22b2f918536e819d33f0a subpackages: - codes - credentials diff --git a/glide.yaml b/glide.yaml index 30251737..0acdb931 100644 --- a/glide.yaml +++ b/glide.yaml @@ -79,6 +79,10 @@ import: - compute/v0.beta - googleapi +### DigitalOcean + - package: github.com/digitalocean/godo + version: 2ff8a02a86cd6918b384a5000ceebe886844fbce + ################################################################################ ## Build System Tools ## ################################################################################ diff --git a/imports/executors/imports_executor.go b/imports/executors/imports_executor.go index ecad60e2..34deffd2 100644 --- a/imports/executors/imports_executor.go +++ b/imports/executors/imports_executor.go @@ -4,6 +4,7 @@ package executors import ( // load the storage executors + _ "github.com/codedellemc/libstorage/drivers/storage/digitalocean/executor" _ "github.com/codedellemc/libstorage/drivers/storage/ebs/executor" _ "github.com/codedellemc/libstorage/drivers/storage/efs/executor" _ "github.com/codedellemc/libstorage/drivers/storage/gcepd/executor" diff --git a/imports/executors/imports_executor_digitalocean.go b/imports/executors/imports_executor_digitalocean.go new file mode 100644 index 00000000..d2fa9919 --- /dev/null +++ b/imports/executors/imports_executor_digitalocean.go @@ -0,0 +1,8 @@ +// +build libstorage_storage_executor,libstorage_storage_executor_digitalocean + +package executors + +import ( + // load the packages + _ "github.com/codedellemc/libstorage/drivers/storage/digitalocean/executor" +) diff --git a/imports/remote/imports_remote.go b/imports/remote/imports_remote.go index b072b01c..19e6436c 100644 --- a/imports/remote/imports_remote.go +++ b/imports/remote/imports_remote.go @@ -4,6 +4,7 @@ package remote import ( // import to load + _ "github.com/codedellemc/libstorage/drivers/storage/digitalocean/storage" _ "github.com/codedellemc/libstorage/drivers/storage/ebs/storage" _ "github.com/codedellemc/libstorage/drivers/storage/efs/storage" _ "github.com/codedellemc/libstorage/drivers/storage/gcepd/storage" diff --git a/imports/remote/imports_remote_digitalocean.go b/imports/remote/imports_remote_digitalocean.go new file mode 100644 index 00000000..9f9e454c --- /dev/null +++ b/imports/remote/imports_remote_digitalocean.go @@ -0,0 +1,8 @@ +// +build libstorage_storage_driver,libstorage_storage_driver_digitalocean + +package remote + +import ( + // load the packages + _ "github.com/codedellemc/libstorage/drivers/storage/digitalocean/storage" +)