Skip to content
This repository has been archived by the owner on Jan 7, 2025. It is now read-only.

Commit

Permalink
Try gracefully shutting down before deleting
Browse files Browse the repository at this point in the history
Signed-off-by: Johan Siebens <[email protected]>
  • Loading branch information
jsiebens committed May 2, 2021
1 parent ab34c89 commit 489067f
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 20 deletions.
40 changes: 23 additions & 17 deletions plugin/digitalocean.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io/ioutil"
"strconv"
"time"

"github.com/digitalocean/godo"
Expand All @@ -27,12 +28,12 @@ type dropletTemplate struct {
userData string
}

func (t *TargetPlugin) scaleOut(ctx context.Context, num int64, template *dropletTemplate, config map[string]string) error {
log := t.logger.With("action", "scale_out", "tag", template.nodeClass, "count", num)
func (t *TargetPlugin) scaleOut(ctx context.Context, desired, diff int64, template *dropletTemplate, config map[string]string) error {
log := t.logger.With("action", "scale_out", "tag", template.nodeClass, "count", diff)

log.Debug("creating DigitalOcean droplets")

for i := int64(0); i < num; i++ {
for i := int64(0); i < diff; i++ {
createRequest := &godo.DropletCreateRequest{
Name: template.nodeClass + "-" + randstr.String(6),
Region: template.region,
Expand Down Expand Up @@ -65,7 +66,7 @@ func (t *TargetPlugin) scaleOut(ctx context.Context, num int64, template *drople

log.Debug("successfully created DigitalOcean droplets")

if err := t.ensureDropletsAreStable(ctx, template); err != nil {
if err := t.ensureDropletsAreStable(ctx, template, desired); err != nil {
return fmt.Errorf("failed to confirm scale out DigitalOcean droplets: %v", err)
}

Expand All @@ -74,9 +75,9 @@ func (t *TargetPlugin) scaleOut(ctx context.Context, num int64, template *drople
return nil
}

func (t *TargetPlugin) scaleIn(ctx context.Context, num int64, template *dropletTemplate, config map[string]string) error {
func (t *TargetPlugin) scaleIn(ctx context.Context, desired, diff int64, template *dropletTemplate, config map[string]string) error {

ids, err := t.clusterUtils.RunPreScaleInTasks(ctx, config, int(num))
ids, err := t.clusterUtils.RunPreScaleInTasks(ctx, config, int(diff))
if err != nil {
return fmt.Errorf("failed to perform pre-scale Nomad scale in tasks: %v", err)
}
Expand All @@ -98,9 +99,9 @@ func (t *TargetPlugin) scaleIn(ctx context.Context, num int64, template *droplet
return fmt.Errorf("failed to delete instances: %v", err)
}

log.Debug("successfully deleted DigitalOcean droplets")
log.Debug("successfully started deletion process")

if err := t.ensureDropletsAreStable(ctx, template); err != nil {
if err := t.ensureDropletsAreStable(ctx, template, desired); err != nil {
return fmt.Errorf("failed to confirm scale in DigitalOcean droplets: %v", err)
}

Expand All @@ -114,11 +115,11 @@ func (t *TargetPlugin) scaleIn(ctx context.Context, num int64, template *droplet
return nil
}

func (t *TargetPlugin) ensureDropletsAreStable(ctx context.Context, template *dropletTemplate) error {
func (t *TargetPlugin) ensureDropletsAreStable(ctx context.Context, template *dropletTemplate, desired int64) error {

f := func(ctx context.Context) (bool, error) {
total, active, err := t.countDroplets(ctx, template)
if total == active || err != nil {
_, active, err := t.countDroplets(ctx, template)
if desired == active || err != nil {
return true, err
} else {
return false, fmt.Errorf("waiting for droplets to become stable")
Expand All @@ -130,6 +131,7 @@ func (t *TargetPlugin) ensureDropletsAreStable(ctx context.Context, template *dr

func (t *TargetPlugin) deleteDroplets(ctx context.Context, tag string, instanceIDs map[string]bool) error {
// create options. initially, these will be blank
var dropletsToDelete []int
opt := &godo.ListOptions{}
for {
droplets, resp, err := t.client.Droplets.ListByTag(ctx, tag, opt)
Expand All @@ -140,15 +142,19 @@ func (t *TargetPlugin) deleteDroplets(ctx context.Context, tag string, instanceI
for _, d := range droplets {
_, ok := instanceIDs[d.Name]
if ok {
_, err := t.client.Droplets.Delete(ctx, d.ID)
if err != nil {
return err
}
go func(dropletId int) {
log := t.logger.With("action", "delete", "droplet_id", strconv.Itoa(dropletId))
err := shutdownDroplet(dropletId, t.client, log)
if err != nil {
log.Error("error deleting droplet", err)
}
}(d.ID)
dropletsToDelete = append(dropletsToDelete, d.ID)
}
}

// if we are at the last page, break out the for loop
if resp.Links == nil || resp.Links.IsLastPage() {
// if we deleted all droplets or if we are at the last page, break out the for loop
if len(dropletsToDelete) == len(instanceIDs) || resp.Links == nil || resp.Links.IsLastPage() {
break
}

Expand Down
6 changes: 3 additions & 3 deletions plugin/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,13 +123,13 @@ func (t *TargetPlugin) Scale(action sdk.ScalingAction, config map[string]string)
return fmt.Errorf("failed to describe DigitalOcedroplets: %v", err)
}

num, direction := t.calculateDirection(total, action.Count)
diff, direction := t.calculateDirection(total, action.Count)

switch direction {
case "in":
err = t.scaleIn(ctx, num, template, config)
err = t.scaleIn(ctx, action.Count, diff, template, config)
case "out":
err = t.scaleOut(ctx, num, template, config)
err = t.scaleOut(ctx, action.Count, diff, template, config)
default:
t.logger.Info("scaling not required", "tag", template.nodeClass,
"current_count", total, "strategy_count", action.Count)
Expand Down
37 changes: 37 additions & 0 deletions plugin/shutdown.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package plugin

import (
"context"
"fmt"
"time"

"github.com/digitalocean/godo"
"github.com/hashicorp/go-hclog"
)

func shutdownDroplet(
dropletId int,
client *godo.Client,
log hclog.Logger) error {

// Gracefully power off the droplet.
log.Debug("Gracefully shutting down droplet...")
_, _, err := client.DropletActions.PowerOff(context.TODO(), dropletId)
if err != nil {
// If we get an error the first time, actually report it
return fmt.Errorf("error shutting down droplet: %s", err)
}

err = waitForDropletState("off", dropletId, client, log, 5*time.Minute)
if err != nil {
log.Warn("Timeout while waiting to for droplet to become 'off'")
}

log.Debug("Deleting Droplet...")
_, err = client.Droplets.Delete(context.TODO(), dropletId)
if err != nil {
return fmt.Errorf("error deleting droplet: %s", err)
}

return nil
}
60 changes: 60 additions & 0 deletions plugin/wait.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package plugin

import (
"context"
"fmt"
"time"

"github.com/digitalocean/godo"
"github.com/hashicorp/go-hclog"
)

func waitForDropletState(
desiredState string, dropletId int,
client *godo.Client,
log hclog.Logger,
timeout time.Duration) error {
done := make(chan struct{})
defer close(done)

result := make(chan error, 1)
go func() {
attempts := 0
for {
attempts += 1

log.Debug(fmt.Sprintf("Checking droplet status... (attempt: %d)", attempts))
droplet, _, err := client.Droplets.Get(context.TODO(), dropletId)
if err != nil {
result <- err
return
}

if droplet.Status == desiredState {
result <- nil
return
}

// Wait 3 seconds in between
time.Sleep(3 * time.Second)

// Verify we shouldn't exit
select {
case <-done:
// We finished, so just exit the goroutine
return
default:
// Keep going
}
}
}()

log.Debug(fmt.Sprintf("Waiting for up to %d seconds for droplet to become %s", timeout/time.Second, desiredState))
select {
case err := <-result:
return err
case <-time.After(timeout):
err := fmt.Errorf("timeout while waiting to for droplet to become '%s'", desiredState)
return err
}
}

0 comments on commit 489067f

Please sign in to comment.