Improve graceful shutdown:

- (Re-)Introduce Shutdown Readiness check
- Always tear down Shutdown Readiness check during shutdown
- Add optional delay between pre-shutdown phase and shutdown phase to give the infrastructure time to stop routing traffic to the application

core/runtime/src/main/java/io/quarkus/runtime/shutdown/ShutdownConfig.java

@@ -18,8 +18,21 @@ public class ShutdownConfig {
     @ConfigItem
     public Optional<Duration> timeout;
 
+    /**
+     * Delay between shutdown being requested and actually initiated. Also called the pre-shutdown phase.
+     * In pre-shutdown, the server continues working as usual, except a readiness probe starts reporting "down"
+     * (if the {@code smallrye-health} extension is present). This gives the infrastructure time to detect
+     * that the application instance is shutting down and stop routing traffic to it.
+     */
+    @ConfigItem
+    public Optional<Duration> delay;
+
     public boolean isShutdownTimeoutSet() {
         return timeout.isPresent() && timeout.get().toMillis() > 0;
     }
 
+    public boolean isDelaySet() {
+        return delay.isPresent() && delay.get().toMillis() > 0;
+    }
+
 }

core/runtime/src/main/java/io/quarkus/runtime/shutdown/ShutdownRecorder.java

@@ -1,6 +1,5 @@
 package io.quarkus.runtime.shutdown;
 
-import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CountDownLatch;
@@ -16,43 +15,52 @@ public class ShutdownRecorder {
     private static final Logger log = Logger.getLogger(ShutdownRecorder.class);
 
     private static volatile List<ShutdownListener> shutdownListeners;
-    private static volatile Optional<Duration> waitTime;
-
-    final ShutdownConfig shutdownConfig;
+    private static volatile ShutdownConfig shutdownConfig;
 
     public ShutdownRecorder(ShutdownConfig shutdownConfig) {
-        this.shutdownConfig = shutdownConfig;
+        ShutdownRecorder.shutdownConfig = shutdownConfig;
     }
 
     public void setListeners(List<ShutdownListener> listeners) {
-        shutdownListeners = listeners;
-        waitTime = shutdownConfig.timeout;
+        shutdownListeners = Optional.ofNullable(listeners).orElseGet(List::of);
     }
 
     public static void runShutdown() {
-        if (shutdownListeners == null) {
-            return;
-        }
-        log.debug("Attempting to gracefully shutdown.");
         try {
-            CountDownLatch preShutdown = new CountDownLatch(shutdownListeners.size());
-            for (ShutdownListener i : shutdownListeners) {
-                i.preShutdown(new LatchShutdownNotification(preShutdown));
-            }
+            executePreShutdown();
+            waitForDelay();
+            executeShutdown();
+        } catch (Throwable e) {
+            log.error("Graceful shutdown failed", e);
+        }
+    }
 
-            preShutdown.await();
-            CountDownLatch shutdown = new CountDownLatch(shutdownListeners.size());
-            for (ShutdownListener i : shutdownListeners) {
-                i.shutdown(new LatchShutdownNotification(shutdown));
-            }
-            if (waitTime.isPresent()) {
-                if (!shutdown.await(waitTime.get().toMillis(), TimeUnit.MILLISECONDS)) {
-                    log.error("Timed out waiting for graceful shutdown, shutting down anyway.");
-                }
+    private static void executePreShutdown() throws InterruptedException {
+        CountDownLatch preShutdown = new CountDownLatch(shutdownListeners.size());
+        for (ShutdownListener i : shutdownListeners) {
+            i.preShutdown(new LatchShutdownNotification(preShutdown));
+        }
+        preShutdown.await();
+    }
+
+    private static void waitForDelay() {
+        if (shutdownConfig.isDelaySet()) {
+            try {
+                Thread.sleep(shutdownConfig.delay.get().toMillis());
+            } catch (InterruptedException e) {
+                log.error("Interrupted while waiting for delay, continuing to shutdown immediately");
             }
+        }
+    }
 
-        } catch (Throwable e) {
-            log.error("Graceful shutdown failed", e);
+    private static void executeShutdown() throws InterruptedException {
+        CountDownLatch shutdown = new CountDownLatch(shutdownListeners.size());
+        for (ShutdownListener i : shutdownListeners) {
+            i.shutdown(new LatchShutdownNotification(shutdown));
+        }
+        if (shutdownConfig.isShutdownTimeoutSet()
+                && !shutdown.await(shutdownConfig.timeout.get().toMillis(), TimeUnit.MILLISECONDS)) {
+            log.error("Timed out waiting for graceful shutdown, shutting down anyway.");
         }
     }
 

extensions/smallrye-health/deployment/src/main/java/io/quarkus/smallrye/health/deployment/SmallRyeHealthProcessor.java

@@ -9,7 +9,6 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.BooleanSupplier;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.eclipse.microprofile.config.Config;
@@ -54,6 +53,7 @@
 import io.quarkus.runtime.configuration.ConfigurationException;
 import io.quarkus.smallrye.health.deployment.spi.HealthBuildItem;
 import io.quarkus.smallrye.health.runtime.QuarkusAsyncHealthCheckFactory;
+import io.quarkus.smallrye.health.runtime.ShutdownReadinessCheck;
 import io.quarkus.smallrye.health.runtime.ShutdownReadinessListener;
 import io.quarkus.smallrye.health.runtime.SmallRyeHealthGroupHandler;
 import io.quarkus.smallrye.health.runtime.SmallRyeHealthHandler;
@@ -82,6 +82,8 @@
 class SmallRyeHealthProcessor {
     private static final Logger LOG = Logger.getLogger(SmallRyeHealthProcessor.class);
 
+    private static final String CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED = "quarkus.smallrye-health.management.enabled";
+
     private static final DotName LIVENESS = DotName.createSimple(Liveness.class.getName());
     private static final DotName READINESS = DotName.createSimple(Readiness.class.getName());
     private static final DotName STARTUP = DotName.createSimple(Startup.class.getName());
@@ -132,7 +134,7 @@ List<HotDeploymentWatchedFileBuildItem> brandingFiles() {
                 BRANDING_LOGO_MODULE,
                 BRANDING_STYLE_MODULE,
                 BRANDING_FAVICON_MODULE).map(HotDeploymentWatchedFileBuildItem::new)
-                .collect(Collectors.toList());
+                .toList();
     }
 
     @BuildStep
@@ -211,7 +213,7 @@ public void defineHealthRoutes(BuildProducer<RouteBuildItem> routes,
 
         // Register the health handler
         routes.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                .management("quarkus.smallrye-health.management.enabled")
+                .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                 .route(healthConfig.rootPath)
                 .routeConfigKey("quarkus.smallrye-health.root-path")
                 .handler(new SmallRyeHealthHandler())
@@ -220,47 +222,47 @@ public void defineHealthRoutes(BuildProducer<RouteBuildItem> routes,
 
         // Register the liveness handler
         routes.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                .management("quarkus.smallrye-health.management.enabled")
+                .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                 .nestedRoute(healthConfig.rootPath, healthConfig.livenessPath)
                 .handler(new SmallRyeLivenessHandler())
                 .displayOnNotFoundPage()
                 .build());
 
         // Register the readiness handler
         routes.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                .management("quarkus.smallrye-health.management.enabled")
+                .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                 .nestedRoute(healthConfig.rootPath, healthConfig.readinessPath)
                 .handler(new SmallRyeReadinessHandler())
                 .displayOnNotFoundPage()
                 .build());
 
         // Register the health group handlers
         routes.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                .management("quarkus.smallrye-health.management.enabled")
+                .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                 .nestedRoute(healthConfig.rootPath, healthConfig.groupPath)
                 .handler(new SmallRyeHealthGroupHandler())
                 .displayOnNotFoundPage()
                 .build());
 
         SmallRyeIndividualHealthGroupHandler handler = new SmallRyeIndividualHealthGroupHandler();
         routes.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                .management("quarkus.smallrye-health.management.enabled")
+                .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                 .nestedRoute(healthConfig.rootPath, healthConfig.groupPath + "/*")
                 .handler(handler)
                 .displayOnNotFoundPage()
                 .build());
 
         // Register the wellness handler
         routes.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                .management("quarkus.smallrye-health.management.enabled")
+                .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                 .nestedRoute(healthConfig.rootPath, healthConfig.wellnessPath)
                 .handler(new SmallRyeWellnessHandler())
                 .displayOnNotFoundPage()
                 .build());
 
         // Register the startup handler
         routes.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                .management("quarkus.smallrye-health.management.enabled")
+                .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                 .nestedRoute(healthConfig.rootPath, healthConfig.startupPath)
                 .handler(new SmallRyeStartupHandler())
                 .displayOnNotFoundPage()
@@ -314,10 +316,9 @@ private void warnIfJaxRsPathUsed(IndexView index, DotName healthAnnotation) {
                 if (target.asClass().declaredAnnotation(JAX_RS_PATH) != null) {
                     containsPath = true;
                 }
-            } else if (target.kind() == Kind.METHOD) {
-                if (target.asMethod().hasAnnotation(JAX_RS_PATH)) {
-                    containsPath = true;
-                }
+            } else if (target.kind() == Kind.METHOD && target.asMethod().hasAnnotation(JAX_RS_PATH)) {
+                containsPath = true;
+
             }
             if (containsPath) {
                 LOG.warnv(
@@ -355,6 +356,14 @@ public void kubernetes(NonApplicationRootPathBuildItem nonApplicationRootPathBui
                                 healthConfig.startupPath)));
     }
 
+    @BuildStep
+    AdditionalBeanBuildItem shutdownHealthCheck() {
+        return AdditionalBeanBuildItem.builder()
+                .addBeanClass(ShutdownReadinessCheck.class)
+                .setUnremovable()
+                .build();
+    }
+
     @BuildStep
     ShutdownListenerBuildItem shutdownListener() {
         return new ShutdownListenerBuildItem(new ShutdownReadinessListener());
@@ -413,7 +422,6 @@ void registerHealthUiHandler(
             LaunchModeBuildItem launchMode,
             SmallRyeHealthConfig healthConfig,
             BuildProducer<SmallRyeHealthBuildItem> smallryeHealthBuildProducer, ShutdownContextBuildItem shutdownContext) {
-
         WebJarResultsBuildItem.WebJarResult result = webJarResultsBuildItem.byArtifactKey(HEALTH_UI_WEBJAR_ARTIFACT_KEY);
         if (result == null) {
             return;
@@ -428,15 +436,15 @@ void registerHealthUiHandler(
                     healthUiPath, result.getWebRootConfigurations(), runtimeConfig, shutdownContext);
 
             routeProducer.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                    .management("quarkus.smallrye-health.management.enabled")
+                    .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                     .route(healthConfig.ui.rootPath)
                     .displayOnNotFoundPage("Health UI")
                     .routeConfigKey("quarkus.smallrye-health.ui.root-path")
                     .handler(handler)
                     .build());
 
             routeProducer.produce(nonApplicationRootPathBuildItem.routeBuilder()
-                    .management("quarkus.smallrye-health.management.enabled")
+                    .management(CONFIG_KEY_HEALTH_MANAGEMENT_ENABLED)
                     .route(healthConfig.ui.rootPath + "*")
                     .handler(handler)
                     .build());

extensions/smallrye-health/deployment/src/test/java/io/quarkus/smallrye/health/test/AdditionalJsonPropertiesConfigTest.java

@@ -1,6 +1,6 @@
 package io.quarkus.smallrye.health.test;
 
-import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.is;
 
 import org.jboss.shrinkwrap.api.asset.EmptyAsset;
@@ -11,7 +11,7 @@
 import io.restassured.RestAssured;
 import io.restassured.parsing.Parser;
 
-public class AdditionalJsonPropertiesConfigTest {
+class AdditionalJsonPropertiesConfigTest {
 
     @RegisterExtension
     static final QuarkusUnitTest config = new QuarkusUnitTest()
@@ -22,14 +22,14 @@ public class AdditionalJsonPropertiesConfigTest {
             .overrideConfigKey("quarkus.smallrye-health.additional.property.testProp2", "testValue2");
 
     @Test
-    public void testAdditionalJsonPropertyInclusions() {
+    void testAdditionalJsonPropertyInclusions() {
         try {
             RestAssured.defaultParser = Parser.JSON;
             RestAssured.when().get("/q/health").then()
                     .body("status", is("UP"),
-                            "checks.size()", is(1),
-                            "checks.status", contains("UP"),
-                            "checks.name", contains("basic"),
+                            "checks.size()", is(2),
+                            "checks.status", hasItems("UP", "UP"),
+                            "checks.name", hasItems("basic", "Graceful Shutdown"),
                             "testProp1", is("testValue1"),
                             "testProp2", is("testValue2"));
         } finally {

extensions/smallrye-health/deployment/src/test/java/io/quarkus/smallrye/health/test/AsyncDispatchedThreadTest.java

@@ -1,6 +1,7 @@
 package io.quarkus.smallrye.health.test;
 
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.stringContainsInOrder;
@@ -20,7 +21,7 @@
 import io.smallrye.health.api.AsyncHealthCheck;
 import io.smallrye.mutiny.Uni;
 
-public class AsyncDispatchedThreadTest {
+class AsyncDispatchedThreadTest {
 
     @RegisterExtension
     static final QuarkusUnitTest config = new QuarkusUnitTest()
@@ -29,7 +30,7 @@ public class AsyncDispatchedThreadTest {
                     .addAsManifestResource(EmptyAsset.INSTANCE, "beans.xml"));
 
     @Test
-    public void check() {
+    void check() {
         RestAssured.when().get("/q/health/live").then()
                 .body("status", is("UP"),
                         "checks.status", contains("UP"),
@@ -40,11 +41,14 @@ public void check() {
 
         RestAssured.when().get("/q/health/ready").then()
                 .body("status", is("UP"),
-                        "checks.status", contains("UP"),
-                        "checks.name", contains("my-readiness-check"),
-                        "checks.data.thread[0]", stringContainsInOrder("loop"),
-                        "checks.data.thread[0]", not(stringContainsInOrder("executor-thread")),
-                        "checks.data.request[0]", is(true));
+                        "checks.size()", is(2),
+                        "checks.status", hasItems("UP", "UP"),
+                        "checks.name", hasItems("my-readiness-check", "Graceful Shutdown"),
+                        "checks.findAll { it.name == 'my-readiness-check' }.size()", is(1),
+                        "checks.findAll { it.name = 'my-readiness-check' }.data.thread[0]", stringContainsInOrder("loop"),
+                        "checks.findAll { it.name = 'my-readiness-check' }.data.thread[0]",
+                        not(stringContainsInOrder("executor-thread")),
+                        "checks.findAll { it.name = 'my-readiness-check' }.data.request[0]", is(true));
     }
 
     @ApplicationScoped

extensions/smallrye-health/deployment/src/test/java/io/quarkus/smallrye/health/test/BlockingNonBlockingTest.java

@@ -1,7 +1,7 @@
 package io.quarkus.smallrye.health.test;
 
 import static io.restassured.RestAssured.given;
-import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.is;
 
 import java.time.Duration;
@@ -24,7 +24,7 @@
 import io.smallrye.health.SmallRyeHealthReporter;
 import io.smallrye.mutiny.Uni;
 
-public class BlockingNonBlockingTest {
+class BlockingNonBlockingTest {
 
     @RegisterExtension
     static final QuarkusUnitTest config = new QuarkusUnitTest()
@@ -33,7 +33,7 @@ public class BlockingNonBlockingTest {
                     .addAsManifestResource(EmptyAsset.INSTANCE, "beans.xml"));
 
     @Test
-    public void testRegisterHealthOnBlockingThreadStep1() {
+    void testRegisterHealthOnBlockingThreadStep1() {
         // initial startup health blocking call on worker thread
         given()
                 .when().get("/start-health")
@@ -45,8 +45,8 @@ public void testRegisterHealthOnBlockingThreadStep1() {
             for (int i = 0; i < 3; i++) {
                 RestAssured.when().get("/q/health").then()
                         .body("status", is("UP"),
-                                "checks.status", contains("UP"),
-                                "checks.name", contains("blocking"));
+                                "checks.status", hasItems("UP", "UP"),
+                                "checks.name", hasItems("blocking", "Graceful Shutdown"));
             }
         } finally {
             RestAssured.reset();