Improve Node Shutdown Observability

server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -20,25 +22,42 @@
 import java.util.Objects;
 
 public class ShutdownShardMigrationStatus implements Writeable, ToXContentObject {
+    private static final Version ALLOCATION_DECISION_ADDED_VERSION = Version.V_8_0_0;
 
     private final SingleNodeShutdownMetadata.Status status;
     private final long shardsRemaining;
     @Nullable private final String explanation;
+    @Nullable private final ShardAllocationDecision allocationDecision;
 
     public ShutdownShardMigrationStatus(SingleNodeShutdownMetadata.Status status, long shardsRemaining) {
-        this(status, shardsRemaining, null);
+        this(status, shardsRemaining, null, null);
     }
 
     public ShutdownShardMigrationStatus(SingleNodeShutdownMetadata.Status status, long shardsRemaining, @Nullable String explanation) {
+        this(status, shardsRemaining, explanation, null);
+    }
+
+    public ShutdownShardMigrationStatus(
+        SingleNodeShutdownMetadata.Status status,
+        long shardsRemaining,
+        @Nullable String explanation,
+        @Nullable ShardAllocationDecision allocationDecision
+    ) {
         this.status = Objects.requireNonNull(status, "status must not be null");
         this.shardsRemaining = shardsRemaining;
         this.explanation = explanation;
+        this.allocationDecision = allocationDecision;
     }
 
     public ShutdownShardMigrationStatus(StreamInput in) throws IOException {
         this.status = in.readEnum(SingleNodeShutdownMetadata.Status.class);
         this.shardsRemaining = in.readLong();
         this.explanation = in.readOptionalString();
+        if (in.getVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
+            this.allocationDecision = in.readOptionalWriteable(ShardAllocationDecision::new);
+        } else {
+            this.allocationDecision = null;
+        }
     }
 
     public long getShardsRemaining() {
@@ -61,6 +80,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (Objects.nonNull(explanation)) {
             builder.field("explanation", explanation);
         }
+        if (Objects.nonNull(allocationDecision)) {
+            builder.startObject("node_allocation_decisions");
+            {
+                allocationDecision.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
         builder.endObject();
         return builder;
     }
@@ -70,19 +96,25 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeEnum(status);
         out.writeLong(shardsRemaining);
         out.writeOptionalString(explanation);
+        if (out.getVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
+            out.writeOptionalWriteable(allocationDecision);
+        }
     }
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if ((o instanceof ShutdownShardMigrationStatus) == false) return false;
         ShutdownShardMigrationStatus that = (ShutdownShardMigrationStatus) o;
-        return shardsRemaining == that.shardsRemaining && status == that.status && Objects.equals(explanation, that.explanation);
+        return shardsRemaining == that.shardsRemaining
+            && status == that.status
+            && Objects.equals(explanation, that.explanation)
+            && Objects.equals(allocationDecision, that.allocationDecision);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(status, shardsRemaining, explanation);
+        return Objects.hash(status, shardsRemaining, explanation, allocationDecision);
     }
 
     @Override

x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java

@@ -71,6 +71,8 @@ protected void masterOperation(
             public ClusterState execute(ClusterState currentState) throws Exception {
                 NodesShutdownMetadata currentShutdownMetadata = currentState.metadata().custom(NodesShutdownMetadata.TYPE);
 
+                logger.info("removing shutdown record for node [{}]", request.getNodeId());
+
                 return ClusterState.builder(currentState)
                     .metadata(
                         Metadata.builder(currentState.metadata())

x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java

@@ -262,17 +262,15 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
                 return hasShardCopyOnOtherNode == false;
             })
             .peek(pair -> {
-                if (logger.isTraceEnabled()) { // don't serialize the decision unless we have to
-                    logger.trace(
-                        "node [{}] shutdown of type [{}] stalled: found shard [{}][{}] from index [{}] with negative decision: [{}]",
-                        nodeId,
-                        shutdownType,
-                        pair.v1().getId(),
-                        pair.v1().primary() ? "primary" : "replica",
-                        pair.v1().shardId().getIndexName(),
-                        Strings.toString(pair.v2())
-                    );
-                }
+                logger.debug(
+                    "node [{}] shutdown of type [{}] stalled: found shard [{}][{}] from index [{}] with negative decision: [{}]",
+                    nodeId,
+                    shutdownType,
+                    pair.v1().getId(),
+                    pair.v1().primary() ? "primary" : "replica",
+                    pair.v1().shardId().getIndexName(),
+                    Strings.toString(pair.v2())
+                );
             })
             .findFirst();
 
@@ -287,6 +285,7 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
         } else if (unmovableShard.isPresent()) {
             // We found a shard that can't be moved, so shard relocation is stalled. Blame the unmovable shard.
             ShardRouting shardRouting = unmovableShard.get().v1();
+            ShardAllocationDecision decision = unmovableShard.get().v2();
 
             return new ShutdownShardMigrationStatus(
                 SingleNodeShutdownMetadata.Status.STALLED,
@@ -296,7 +295,8 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
                     shardRouting.shardId().getId(),
                     shardRouting.primary() ? "primary" : "replica",
                     shardRouting.index().getName()
-                ).getFormattedMessage()
+                ).getFormattedMessage(),
+                decision
             );
         } else {
             return new ShutdownShardMigrationStatus(SingleNodeShutdownMetadata.Status.IN_PROGRESS, totalRemainingShards);

x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java

@@ -80,6 +80,13 @@ public ClusterState execute(ClusterState currentState) {
                         request.getType(),
                         request.getReason()
                     );
+                } else {
+                    logger.info(
+                        "creating shutdown record for node [{}] of type [{}] with reason [{}]",
+                        request.getNodeId(),
+                        request.getType(),
+                        request.getReason()
+                    );
                 }
 
                 final boolean nodeSeen = currentState.getNodes().nodeExists(request.getNodeId());