Grace period parameter for sigterm shutdown api (#96004)

Adds the ability to pass in the grace period to `PutNodeShutdown` for `SIGTERM` shutdowns.

The parameter is required if the type is `SIGTERM` and not allowed otherwise.

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -123,12 +123,13 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId
     public static final TransportVersion V_8_500_000 = registerTransportVersion(8_500_000, "dc3cbf06-3ed5-4e1b-9978-ee1d04d235bc");
     public static final TransportVersion V_8_500_001 = registerTransportVersion(8_500_001, "c943cfe5-c89d-4eae-989f-f5f4537e84e0");
     public static final TransportVersion V_8_500_002 = registerTransportVersion(8_500_002, "055dd314-ff40-4313-b4c6-9fccddfa42a8");
+    public static final TransportVersion V_8_500_003 = registerTransportVersion(8_500_003, "30adbe0c-8614-40dd-81b5-44e9c657bb77");
 
     /**
      * Reference to the most recent transport version.
      * This should be the transport version with the highest id.
      */
-    public static final TransportVersion CURRENT = V_8_500_002;
+    public static final TransportVersion CURRENT = V_8_500_003;
 
     /**
      * Reference to the earliest compatible transport version to this version of the codebase.

server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java

@@ -35,6 +35,7 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
 
     public static final TransportVersion REPLACE_SHUTDOWN_TYPE_ADDED_VERSION = TransportVersion.V_7_16_0;
     public static final TransportVersion SIGTERM_ADDED_VERSION = TransportVersion.V_8_9_0;
+    public static final TransportVersion GRACE_PERIOD_ADDED_VERSION = TransportVersion.V_8_500_003;
 
     public static final ParseField NODE_ID_FIELD = new ParseField("node_id");
     public static final ParseField TYPE_FIELD = new ParseField("type");
@@ -44,6 +45,7 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
     public static final ParseField ALLOCATION_DELAY_FIELD = new ParseField("allocation_delay");
     public static final ParseField NODE_SEEN_FIELD = new ParseField("node_seen");
     public static final ParseField TARGET_NODE_NAME_FIELD = new ParseField("target_node_name");
+    public static final ParseField GRACE_PERIOD_FIELD = new ParseField("grace_period");
 
     public static final ConstructingObjectParser<SingleNodeShutdownMetadata, Void> PARSER = new ConstructingObjectParser<>(
         "node_shutdown_info",
@@ -54,7 +56,8 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
             (long) a[3],
             (boolean) a[4],
             (TimeValue) a[5],
-            (String) a[6]
+            (String) a[6],
+            (TimeValue) a[7]
         )
     );
 
@@ -71,6 +74,12 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
             ObjectParser.ValueType.STRING_OR_NULL
         );
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TARGET_NODE_NAME_FIELD);
+        PARSER.declareField(
+            ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), GRACE_PERIOD_FIELD.getPreferredName()),
+            GRACE_PERIOD_FIELD,
+            ObjectParser.ValueType.STRING_OR_NULL
+        );
     }
 
     public static SingleNodeShutdownMetadata parse(XContentParser parser) {
@@ -88,6 +97,8 @@ public static SingleNodeShutdownMetadata parse(XContentParser parser) {
     private final TimeValue allocationDelay;
     @Nullable
     private final String targetNodeName;
+    @Nullable
+    private final TimeValue gracePeriod;
 
     /**
      * @param nodeId The node ID that this shutdown metadata refers to.
@@ -102,7 +113,8 @@ private SingleNodeShutdownMetadata(
         long startedAtMillis,
         boolean nodeSeen,
         @Nullable TimeValue allocationDelay,
-        @Nullable String targetNodeName
+        @Nullable String targetNodeName,
+        @Nullable TimeValue gracePeriod
     ) {
         this.nodeId = Objects.requireNonNull(nodeId, "node ID must not be null");
         this.type = Objects.requireNonNull(type, "shutdown type must not be null");
@@ -125,6 +137,20 @@ private SingleNodeShutdownMetadata(
             throw new IllegalArgumentException("target node name is required for REPLACE type shutdowns");
         }
         this.targetNodeName = targetNodeName;
+        if (Type.SIGTERM.equals(type)) {
+            if (gracePeriod == null) {
+                throw new IllegalArgumentException("grace period is required for SIGTERM shutdowns");
+            }
+        } else if (gracePeriod != null) {
+            throw new IllegalArgumentException(
+                format(
+                    "grace period is only valid for SIGTERM type shutdowns, but was given type [%s] and target node name [%s]",
+                    type,
+                    targetNodeName
+                )
+            );
+        }
+        this.gracePeriod = gracePeriod;
     }
 
     public SingleNodeShutdownMetadata(StreamInput in) throws IOException {
@@ -139,6 +165,11 @@ public SingleNodeShutdownMetadata(StreamInput in) throws IOException {
         } else {
             this.targetNodeName = null;
         }
+        if (in.getTransportVersion().onOrAfter(GRACE_PERIOD_ADDED_VERSION)) {
+            this.gracePeriod = in.readOptionalTimeValue();
+        } else {
+            this.gracePeriod = null;
+        }
     }
 
     /**
@@ -197,6 +228,14 @@ public TimeValue getAllocationDelay() {
         return null;
     }
 
+    /**
+     * @return the timeout for a graceful shutdown for a SIGTERM type.
+     */
+    @Nullable
+    public TimeValue getGracePeriod() {
+        return gracePeriod;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeId);
@@ -213,6 +252,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
             out.writeOptionalString(targetNodeName);
         }
+        if (out.getTransportVersion().onOrAfter(GRACE_PERIOD_ADDED_VERSION)) {
+            out.writeOptionalTimeValue(gracePeriod);
+        }
     }
 
     @Override
@@ -230,6 +272,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (targetNodeName != null) {
                 builder.field(TARGET_NODE_NAME_FIELD.getPreferredName(), targetNodeName);
             }
+            if (gracePeriod != null) {
+                builder.field(GRACE_PERIOD_FIELD.getPreferredName(), gracePeriod.getStringRep());
+            }
         }
         builder.endObject();
 
@@ -242,17 +287,27 @@ public boolean equals(Object o) {
         if ((o instanceof SingleNodeShutdownMetadata) == false) return false;
         SingleNodeShutdownMetadata that = (SingleNodeShutdownMetadata) o;
         return getStartedAtMillis() == that.getStartedAtMillis()
+            && getNodeSeen() == that.getNodeSeen()
             && getNodeId().equals(that.getNodeId())
             && getType() == that.getType()
             && getReason().equals(that.getReason())
-            && getNodeSeen() == that.getNodeSeen()
-            && Objects.equals(allocationDelay, that.allocationDelay)
-            && Objects.equals(targetNodeName, that.targetNodeName);
+            && Objects.equals(getAllocationDelay(), that.getAllocationDelay())
+            && Objects.equals(getTargetNodeName(), that.getTargetNodeName())
+            && Objects.equals(getGracePeriod(), that.getGracePeriod());
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(getNodeId(), getType(), getReason(), getStartedAtMillis(), getNodeSeen(), allocationDelay, targetNodeName);
+        return Objects.hash(
+            getNodeId(),
+            getType(),
+            getReason(),
+            getStartedAtMillis(),
+            getNodeSeen(),
+            getAllocationDelay(),
+            getTargetNodeName(),
+            getGracePeriod()
+        );
     }
 
     @Override
@@ -273,6 +328,9 @@ public String toString() {
         if (targetNodeName != null) {
             stringBuilder.append(", targetNodeName=[").append(targetNodeName).append("]");
         }
+        if (gracePeriod != null) {
+            stringBuilder.append(", gracePeriod=[").append(gracePeriod).append("]");
+        }
         stringBuilder.append("}");
         return stringBuilder.toString();
     }
@@ -301,6 +359,7 @@ public static class Builder {
         private boolean nodeSeen = false;
         private TimeValue allocationDelay;
         private String targetNodeName;
+        private TimeValue gracePeriod;
 
         private Builder() {}
 
@@ -367,12 +426,26 @@ public Builder setTargetNodeName(String targetNodeName) {
             return this;
         }
 
+        public Builder setGracePeriod(TimeValue gracePeriod) {
+            this.gracePeriod = gracePeriod;
+            return this;
+        }
+
         public SingleNodeShutdownMetadata build() {
             if (startedAtMillis == -1) {
                 throw new IllegalArgumentException("start timestamp must be set");
             }
 
-            return new SingleNodeShutdownMetadata(nodeId, type, reason, startedAtMillis, nodeSeen, allocationDelay, targetNodeName);
+            return new SingleNodeShutdownMetadata(
+                nodeId,
+                type,
+                reason,
+                startedAtMillis,
+                nodeSeen,
+                allocationDelay,
+                targetNodeName,
+                gracePeriod
+            );
         }
     }
 

server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.node.TestDiscoveryNode;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
@@ -79,6 +80,11 @@ public void testIsNodeShuttingDown() {
                         .setReason("shutdown for a unit test")
                         .setType(type)
                         .setStartedAtMillis(randomNonNegativeLong())
+                        .setGracePeriod(
+                            type == SingleNodeShutdownMetadata.Type.SIGTERM
+                                ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName())
+                                : null
+                        )
                         .build()
                 )
             );
@@ -106,11 +112,14 @@ public void testSigtermIsRemoveInOlderVersions() throws IOException {
             .setType(SingleNodeShutdownMetadata.Type.SIGTERM)
             .setReason("myReason")
             .setStartedAtMillis(0L)
+            .setGracePeriod(new TimeValue(1_000))
             .build();
         BytesStreamOutput out = new BytesStreamOutput();
         out.setTransportVersion(TransportVersion.V_8_7_1);
         metadata.writeTo(out);
-        assertThat(new SingleNodeShutdownMetadata(out.bytes().streamInput()).getType(), equalTo(SingleNodeShutdownMetadata.Type.REMOVE));
+        StreamInput in = out.bytes().streamInput();
+        in.setTransportVersion(TransportVersion.V_8_7_1);
+        assertThat(new SingleNodeShutdownMetadata(in).getType(), equalTo(SingleNodeShutdownMetadata.Type.REMOVE));
 
         out = new BytesStreamOutput();
         metadata.writeTo(out);
@@ -150,6 +159,8 @@ private SingleNodeShutdownMetadata randomNodeShutdownInfo() {
             builder.setAllocationDelay(TimeValue.parseTimeValue(randomTimeValue(), this.getTestName()));
         } else if (type.equals(SingleNodeShutdownMetadata.Type.REPLACE)) {
             builder.setTargetNodeName(randomAlphaOfLengthBetween(5, 10));
+        } else if (type.equals(SingleNodeShutdownMetadata.Type.SIGTERM)) {
+            builder.setGracePeriod(TimeValue.parseTimeValue(randomTimeValue(), this.getTestName()));
         }
         return builder.setNodeSeen(randomBoolean()).build();
     }

server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java

@@ -636,6 +636,9 @@ public void testRemainingDelayCalculationsWithUnrelatedShutdowns() throws Except
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(type)
                 .setTargetNodeName(targetNodeName)
+                .setGracePeriod(
+                    type == SingleNodeShutdownMetadata.Type.SIGTERM ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName()) : null
+                )
                 .build();
             shutdowns.put(shutdown.getNodeId(), shutdown);
         }
@@ -657,6 +660,9 @@ public void testRemainingDelayCalculationWhenNodeIsShuttingDownForRemoval() thro
                 .setReason(this.getTestName())
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(type)
+                .setGracePeriod(
+                    type == SingleNodeShutdownMetadata.Type.SIGTERM ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName()) : null
+                )
                 .build();
             shutdowns.put(shutdown.getNodeId(), shutdown);
 

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.EmptySnapshotsInfoService;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
@@ -210,6 +211,11 @@ private ClusterState prepareState(ClusterState initialState, SingleNodeShutdownM
             .setReason(this.getTestName())
             .setStartedAtMillis(1L)
             .setTargetNodeName(targetNodeName)
+            .setGracePeriod(
+                shutdownType == SingleNodeShutdownMetadata.Type.SIGTERM
+                    ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName())
+                    : null
+            )
             .build();
         NodesShutdownMetadata nodesShutdownMetadata = new NodesShutdownMetadata(new HashMap<>()).putSingleNodeMetadata(
             nodeShutdownMetadata

server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeExecutorTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
@@ -172,6 +173,11 @@ private ClusterState stateWithNodeShuttingDown(ClusterState clusterState, Single
                     .setReason("shutdown for a unit test")
                     .setType(type)
                     .setStartedAtMillis(randomNonNegativeLong())
+                    .setGracePeriod(
+                        type == SingleNodeShutdownMetadata.Type.SIGTERM
+                            ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName())
+                            : null
+                    )
                     .build()
             )
         );

server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java

@@ -624,6 +624,11 @@ public void testTasksNotAssignedToShuttingDownNodes() {
                             .setReason("shutdown for a unit test")
                             .setType(type)
                             .setStartedAtMillis(randomNonNegativeLong())
+                            .setGracePeriod(
+                                type == SingleNodeShutdownMetadata.Type.SIGTERM
+                                    ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName())
+                                    : null
+                            )
                             .build()
                     )
                 );

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.node.Node;
@@ -462,6 +463,9 @@ public void testStepCompletableIfAllShardsActive() {
             Map<String, IndexMetadata> indices = Map.of(index.getName(), indexMetadata);
 
             final String targetNodeName = type == SingleNodeShutdownMetadata.Type.REPLACE ? randomAlphaOfLengthBetween(10, 20) : null;
+            final TimeValue grace = type == SingleNodeShutdownMetadata.Type.SIGTERM
+                ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName())
+                : null;
             ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
                 .metadata(
                     Metadata.builder()
@@ -477,6 +481,7 @@ public void testStepCompletableIfAllShardsActive() {
                                         .setReason("test")
                                         .setNodeId("node1")
                                         .setTargetNodeName(targetNodeName)
+                                        .setGracePeriod(grace)
                                         .build()
                                 )
                             )
@@ -537,6 +542,9 @@ public void testStepBecomesUncompletable() {
             Map<String, IndexMetadata> indices = Map.of(index.getName(), indexMetadata);
 
             final String targetNodeName = type == SingleNodeShutdownMetadata.Type.REPLACE ? randomAlphaOfLengthBetween(10, 20) : null;
+            final TimeValue grace = type == SingleNodeShutdownMetadata.Type.SIGTERM
+                ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName())
+                : null;
             ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
                 .metadata(
                     Metadata.builder()
@@ -552,6 +560,7 @@ public void testStepBecomesUncompletable() {
                                         .setReason("test")
                                         .setNodeId("node1")
                                         .setTargetNodeName(targetNodeName)
+                                        .setGracePeriod(grace)
                                         .build()
                                 )
                             )

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java

@@ -652,6 +652,9 @@ public void testIndicesOnShuttingDownNodesInDangerousStep() {
             );
 
             final String targetNodeName = type == SingleNodeShutdownMetadata.Type.REPLACE ? randomAlphaOfLengthBetween(10, 20) : null;
+            final TimeValue grace = type == SingleNodeShutdownMetadata.Type.SIGTERM
+                ? TimeValue.parseTimeValue(randomTimeValue(), this.getTestName())
+                : null;
             state = ClusterState.builder(state)
                 .metadata(
                     Metadata.builder(state.metadata())
@@ -666,6 +669,7 @@ public void testIndicesOnShuttingDownNodesInDangerousStep() {
                                         .setStartedAtMillis(randomNonNegativeLong())
                                         .setType(type)
                                         .setTargetNodeName(targetNodeName)
+                                        .setGracePeriod(grace)
                                         .build()
                                 )
                             )