Add Azure VMSS target plugin

GNUmakefile

@@ -104,5 +104,11 @@ bin/plugins/datadog:
 	@cd ./plugins/builtin/apm/datadog && go build -o ../../../../$@
 	@echo "==> Done"
 
+bin/plugins/azure-vmss:
+	@echo "==> Building $@"
+	@mkdir -p $$(dirname $@)
+	@cd ./plugins/builtin/target/azure-vmss && go build -o ../../../../$@
+	@echo "==> Done"
+
 .PHONY: plugins
-plugins: bin/plugins/nomad-apm bin/plugins/nomad-target bin/plugins/prometheus bin/plugins/target-value bin/plugins/aws-asg bin/plugins/datadog
+plugins: bin/plugins/nomad-apm bin/plugins/nomad-target bin/plugins/prometheus bin/plugins/target-value bin/plugins/aws-asg bin/plugins/datadog bin/plugins/azure-vmss

go.mod

@@ -3,6 +3,12 @@ module github.com/hashicorp/nomad-autoscaler
 go 1.13
 
 require (
+	github.com/Azure/azure-sdk-for-go v44.0.0+incompatible
+	github.com/Azure/go-autorest/autorest v0.11.0
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.0
+	github.com/Azure/go-autorest/autorest/date v0.3.0
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect
 	github.com/DataDog/datadog-api-client-go v1.0.0-beta.7
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/armon/go-metrics v0.3.3

go.sum

@@ -1,4 +1,28 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/Azure/azure-sdk-for-go v44.0.0+incompatible h1:e82Yv2HNpS0kuyeCrV29OPKvEiqfs2/uJHic3/3iKdg=
+github.com/Azure/azure-sdk-for-go v44.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.0 h1:tnO41Uo+/0sxTMFY/U7aKg2abek3JOnnXcuSuba74jI=
+github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
+github.com/Azure/go-autorest/autorest/adal v0.9.0 h1:SigMbuFNuKgc1xcGhaeapbh+8fgsu+GxgDRFyg7f5lM=
+github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
+github.com/Azure/go-autorest/autorest/azure/auth v0.5.0 h1:nSMjYIe24eBYasAIxt859TxyXef/IqoH+8/g4+LmcVs=
+github.com/Azure/go-autorest/autorest/azure/auth v0.5.0/go.mod h1:QRTvSZQpxqm8mSErhnbI+tANIBAKP7B+UIE2z4ypUO0=
+github.com/Azure/go-autorest/autorest/azure/cli v0.4.0 h1:Ml+UCrnlKD+cJmSzrZ/RDcDw86NjkRUpnFh7V5JUhzU=
+github.com/Azure/go-autorest/autorest/azure/cli v0.4.0/go.mod h1:JljT387FplPzBA31vUcvsetLKF3pec5bdAxjVU4kI2s=
+github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.0 h1:z20OWOSG5aCye0HEkDp6TPmP17ZcfeMxPi6HnSALa8c=
+github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk=
+github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
+github.com/Azure/go-autorest/autorest/validation v0.3.0 h1:3I9AAI63HfcLtphd9g39ruUwRI+Ca+z/f36KHPFRUss=
+github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
+github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/DataDog/datadog-api-client-go v1.0.0-beta.7 h1:GGF2JsBg6MTu5nRLKF2NKrxgQl9g0SGjR6Wo7Z5yRwI=
 github.com/DataDog/datadog-api-client-go v1.0.0-beta.7/go.mod h1:RQvzywj9rwl11tiGxX84pe3hZlU9f3Wd/8435jr7M3s=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@@ -44,6 +68,10 @@ github.com/cucumber/messages-go/v12 v12.0.0/go.mod h1:5zuJu21U6rB+BBqyGoQr839a4h
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4=
+github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -241,6 +269,8 @@ github.com/zclconf/go-cty v1.3.1/go.mod h1:YO23e2L18AG+ZYQfSobnY4G65nvwvprPCxBHk
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

plugins/builtin/target/azure-vmss/main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad-autoscaler/plugins"
+	azure "github.com/hashicorp/nomad-autoscaler/plugins/builtin/target/azure-vmss/plugin"
+)
+
+func main() {
+	plugins.Serve(factory)
+}
+
+// factory returns a new instance of the Azure VMSS plugin.
+func factory(log hclog.Logger) interface{} {
+	return azure.NewAzureVMSSPlugin(log)
+}

plugins/builtin/target/azure-vmss/plugin/azure.go

@@ -0,0 +1,174 @@
+package plugin
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure/auth"
+	"github.com/hashicorp/nomad-autoscaler/sdk"
+	"github.com/hashicorp/nomad-autoscaler/sdk/helper/ptr"
+	"github.com/hashicorp/nomad-autoscaler/sdk/helper/scaleutils"
+)
+
+// argsOrEnv allows you to pick an environmental variable for a setting if the arg is not set
+func argsOrEnv(args map[string]string, key, env string) string {
+	if value, ok := args[key]; ok {
+		return value
+	}
+	return os.Getenv(env)
+}
+
+// setupAzureClients takes the passed config mapping and instantiates the
+// required Azure service clients.
+func (t *TargetPlugin) setupAzureClient(config map[string]string) error {
+	var authorizer autorest.Authorizer
+	// check for environmental variables, and use if the argument hasn't been set in config
+	tenantID := argsOrEnv(config, configKeyTenantID, "ARM_TENANT_ID")
+	clientID := argsOrEnv(config, configKeyClientID, "ARM_CLIENT_ID")
+	subscriptionID := argsOrEnv(config, configKeySubscriptionID, "ARM_SUBSCRIPTION_ID")
+	secretKey := argsOrEnv(config, configKeySecretKey, "ARM_CLIENT_SECRET")
+
+	// Try to use the argument and environment provided arguments first, if this fails fall back to the Azure
+	// SDK provided methods
+	if tenantID != "" && clientID != "" && secretKey != "" {
+		var err error
+		authorizer, err = auth.NewClientCredentialsConfig(clientID, secretKey, tenantID).Authorizer()
+		if err != nil {
+			return fmt.Errorf("azure-vmss (ClientCredentials): %s", err)
+		}
+	} else {
+		var err error
+		authorizer, err = auth.NewAuthorizerFromEnvironment()
+		if err != nil {
+			return fmt.Errorf("azure-vmss (EnvironmentCredentials): %s", err)
+		}
+	}
+
+	vmss := compute.NewVirtualMachineScaleSetsClient(subscriptionID)
+	vmss.Sender = autorest.CreateSender()
+	vmss.Authorizer = authorizer
+
+	t.vmss = vmss
+	return nil
+}
+
+// scaleOut updates the Scale Set desired count to match what the
+// Autoscaler has deemed required.
+func (t *TargetPlugin) scaleOut(ctx context.Context, resourceGroup string, vmScaleSet string, count int64) error {
+
+	// Create a logger for this action to pre-populate useful information we
+	// would like on all log lines.
+	log := t.logger.With("action", "scale_out", "vmss_name", vmScaleSet,
+		"desired_count", count)
+
+	future, err := t.vmss.Update(ctx, resourceGroup, vmScaleSet, compute.VirtualMachineScaleSetUpdate{
+		Sku: &compute.Sku{
+			Capacity: ptr.Int64ToPtr(count),
+		},
+	})
+
+	err = future.WaitForCompletionRef(ctx, t.vmss.Client)
+	if err != nil {
+		return fmt.Errorf("cannot get the vmss update future response: %v", err)
+	}
+
+	log.Info("successfully performed and verified scaling out")
+	return nil
+}
+
+// scaleIn drain and delete Scale Set instances to match the Autoscaler has deemed required.
+func (t *TargetPlugin) scaleIn(ctx context.Context, resourceGroup string, vmScaleSet string, num int64, config map[string]string) error {
+
+	scaleReq, err := t.generateScaleReq(num, config)
+	if err != nil {
+		return fmt.Errorf("failed to generate scale in request: %v", err)
+	}
+
+	ids, err := t.scaleInUtils.RunPreScaleInTasks(ctx, scaleReq)
+	if err != nil {
+		return fmt.Errorf("failed to perform Nomad scale in tasks: %v", err)
+	}
+
+	// Grab the instanceIDs once as it is used multiple times throughout the
+	// scale in event.
+	var instanceIDs []string
+	for _, node := range ids {
+
+		// RemoteID should be in the format of "{scale-set-name}_{instance-id}"
+		// If RemoteID doesn't start vmScaleSet then assume its not part of this scale set.
+		// https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-instance-ids#scale-set-vm-names
+		if idx := strings.LastIndex(node.RemoteID, "_"); idx != -1 && strings.EqualFold(node.RemoteID[0:idx], vmScaleSet) {
+			instanceIDs = append(instanceIDs, node.RemoteID[idx+1:])
+		} else {
+			return errors.New("failed to get instance-id from remoteid")
+		}
+	}
+
+	// Create a logger for this action to pre-populate useful information we
+	// would like on all log lines.
+	log := t.logger.With("action", "scale_in", "resource_group", resourceGroup,
+		"vmss_name", vmScaleSet, "instances", instanceIDs)
+
+	// Terminate the detached instances.
+	log.Debug("deleting Azure ScaleSet instances")
+
+	future, err := t.vmss.DeleteInstances(ctx, resourceGroup, vmScaleSet, compute.VirtualMachineScaleSetVMInstanceRequiredIDs{
+		InstanceIds: ptr.StringArrToPtr(instanceIDs),
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to scale in Azure ScaleSet: %v", err)
+	}
+
+	if err := future.WaitForCompletionRef(ctx, t.vmss.Client); err != nil {
+		return fmt.Errorf("failed to scale in Azure ScaleSet: %v", err)
+	}
+
+	log.Info("successfully deleted Azure ScaleSet instances")
+
+	// Run any post scale in tasks that are desired.
+	if err := t.scaleInUtils.RunPostScaleInTasks(config, ids); err != nil {
+		return fmt.Errorf("failed to perform post-scale Nomad scale in tasks: %v", err)
+	}
+
+	return nil
+}
+
+func (t *TargetPlugin) generateScaleReq(num int64, config map[string]string) (*scaleutils.ScaleInReq, error) {
+
+	// Pull the class key from the config mapping. This is a required value and
+	// we cannot scale without this.
+	class, ok := config[sdk.TargetConfigKeyClass]
+	if !ok {
+		return nil, fmt.Errorf("required config param %q not found", sdk.TargetConfigKeyClass)
+	}
+
+	// The drain_deadline is an optional parameter so define out default and
+	// then attempt to find an operator specified value.
+	drain := scaleutils.DefaultDrainDeadline
+
+	if drainString, ok := config[sdk.TargetConfigKeyDrainDeadline]; ok {
+		d, err := time.ParseDuration(drainString)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %q as time duration", drainString)
+		}
+		drain = d
+	}
+
+	return &scaleutils.ScaleInReq{
+		Num:           int(num),
+		DrainDeadline: drain,
+		PoolIdentifier: &scaleutils.PoolIdentifier{
+			IdentifierKey: scaleutils.IdentifierKeyClass,
+			Value:         class,
+		},
+		RemoteProvider: scaleutils.RemoteProviderAzureInstanceID,
+		NodeIDStrategy: scaleutils.IDStrategyNewestCreateIndex,
+	}, nil
+}

plugins/builtin/target/azure-vmss/plugin/azure_test.go

@@ -0,0 +1,85 @@
+package plugin
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad-autoscaler/sdk/helper/scaleutils"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTargetPlugin_generateScaleReq(t *testing.T) {
+	testCases := []struct {
+		inputNum            int64
+		inputConfig         map[string]string
+		expectedOutputReq   *scaleutils.ScaleInReq
+		expectedOutputError error
+		name                string
+	}{
+		{
+			inputNum: 2,
+			inputConfig: map[string]string{
+				"node_class":          "high-memory",
+				"node_drain_deadline": "5m",
+			},
+			expectedOutputReq: &scaleutils.ScaleInReq{
+				Num:           2,
+				DrainDeadline: 5 * time.Minute,
+				PoolIdentifier: &scaleutils.PoolIdentifier{
+					IdentifierKey: scaleutils.IdentifierKeyClass,
+					Value:         "high-memory",
+				},
+				RemoteProvider: scaleutils.RemoteProviderAzureInstanceID,
+				NodeIDStrategy: scaleutils.IDStrategyNewestCreateIndex,
+			},
+			expectedOutputError: nil,
+			name:                "valid request with drain_deadline in config",
+		},
+		{
+			inputNum:            2,
+			inputConfig:         map[string]string{},
+			expectedOutputReq:   nil,
+			expectedOutputError: errors.New("required config param \"node_class\" not found"),
+			name:                "no class key found in config",
+		},
+		{
+			inputNum: 2,
+			inputConfig: map[string]string{
+				"node_class": "high-memory",
+			},
+			expectedOutputReq: &scaleutils.ScaleInReq{
+				Num:           2,
+				DrainDeadline: 15 * time.Minute,
+				PoolIdentifier: &scaleutils.PoolIdentifier{
+					IdentifierKey: scaleutils.IdentifierKeyClass,
+					Value:         "high-memory",
+				},
+				RemoteProvider: scaleutils.RemoteProviderAzureInstanceID,
+				NodeIDStrategy: scaleutils.IDStrategyNewestCreateIndex,
+			},
+			expectedOutputError: nil,
+			name:                "drain_deadline not specified within config",
+		},
+		{
+			inputNum: 2,
+			inputConfig: map[string]string{
+				"node_class":          "high-memory",
+				"node_drain_deadline": "time to make a cuppa",
+			},
+			expectedOutputReq:   nil,
+			expectedOutputError: errors.New("failed to parse \"time to make a cuppa\" as time duration"),
+			name:                "malformed drain_deadline config value",
+		},
+	}
+
+	tp := TargetPlugin{}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actualReq, actualErr := tp.generateScaleReq(tc.inputNum, tc.inputConfig)
+			assert.Equal(t, tc.expectedOutputReq, actualReq, tc.name)
+			assert.Equal(t, tc.expectedOutputError, actualErr, tc.name)
+		})
+	}
+}