Downgrade error message from app scaling due to active deployment

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 IMPROVEMENTS:
  * agent: Dispense `fixed-value`, `pass-through`, and `threshold` strategy plugins by default [[GH-536](https://github.com/hashicorp/nomad-autoscaler/pull/536)]
+ * plugins/target/nomad: Reduce log level for active deployments error messages [[GH-542](https://github.com/hashicorp/nomad-autoscaler/pull/542)]
  * policy: Prevent scaling cluster to zero when using the Nomad APM [[GH-534](https://github.com/hashicorp/nomad-autoscaler/pull/534)]
  * scaleutils: Add combined filter to allow filtering by node class and datacenter [[GH-535](https://github.com/hashicorp/nomad-autoscaler/pull/535)]
 

plugins/builtin/target/nomad/plugin/plugin.go

@@ -2,6 +2,7 @@ package nomad
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -135,6 +136,12 @@ func (t *TargetPlugin) Scale(action sdk.ScalingAction, config map[string]string)
 		&q)
 
 	if err != nil {
+		// Active deployments errors are fairly common and usually not
+		// impactful to the target's eventual end state, so special case them
+		// to return a no-op error instead.
+		if strings.Contains(err.Error(), "job scaling blocked due to active deployment") {
+			return sdk.NewTargetScalingNoOpError("skipping scaling group %s/%s due to active deployment", config[configKeyJobID], config[configKeyGroup])
+		}
 		return fmt.Errorf("failed to scale group %s/%s: %v", config[configKeyJobID], config[configKeyGroup], err)
 	}
 	return nil

policyeval/base_worker.go

@@ -214,6 +214,11 @@ func (w *BaseWorker) handlePolicy(ctx context.Context, eval *sdk.ScalingEvaluati
 	// handler understand what do to.
 	err = w.runTargetScale(targetInst, eval.Policy, *winningAction)
 	if err != nil {
+		if _, ok := err.(*sdk.TargetScalingNoOpError); ok {
+			logger.Info("scaling action skipped", "reason", err)
+			return nil
+		}
+
 		metrics.IncrCounter([]string{"scale", "invoke", "error_count"}, 1)
 		return fmt.Errorf("failed to scale target: %v", err)
 	} else {

sdk/target.go

@@ -1,5 +1,29 @@
 package sdk
 
+import (
+	"fmt"
+)
+
+// TargetScalingNoOp is a special error type that can be used by target plugins
+// to indicate that a scaling request didn't result in any action, but didn't
+// fail either.
+// This can be used to avoid post-scaling actions such as placing the policy in
+// cooldown.
+type TargetScalingNoOpError struct {
+	Err error
+}
+
+// NewTargetScalingNoOpError returns a new target scaling no-op error with the
+// provided formatted message.
+func NewTargetScalingNoOpError(msg string, args ...interface{}) *TargetScalingNoOpError {
+	return &TargetScalingNoOpError{Err: fmt.Errorf(msg, args...)}
+}
+
+// Error implements the error interface.
+func (n *TargetScalingNoOpError) Error() string {
+	return n.Err.Error()
+}
+
 // TargetStatus is the response object when performing the Status call of the
 // target plugin interface. The response details key information about the
 // current state of the target.