diff --git a/reactor-netty-http/src/main/java/reactor/netty/http/server/HttpTrafficHandler.java b/reactor-netty-http/src/main/java/reactor/netty/http/server/HttpTrafficHandler.java
index 9a3fe20a91..2dd846fa18 100644
--- a/reactor-netty-http/src/main/java/reactor/netty/http/server/HttpTrafficHandler.java
+++ b/reactor-netty-http/src/main/java/reactor/netty/http/server/HttpTrafficHandler.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011-2023 VMware, Inc. or its affiliates, All Rights Reserved.
+ * Copyright (c) 2011-2024 VMware, Inc. or its affiliates, All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -154,7 +154,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) {
 		}
 		// read message and track if it was keepAlive
 		if (msg instanceof HttpRequest) {
-			IdleTimeoutHandler.removeIdleTimeoutHandler(ctx.pipeline());
+			if (idleTimeout != null) {
+				IdleTimeoutHandler.removeIdleTimeoutHandler(ctx.pipeline());
+			}
 
 			final HttpRequest request = (HttpRequest) msg;
 
diff --git a/reactor-netty-http/src/main/java/reactor/netty/http/server/IdleTimeoutHandler.java b/reactor-netty-http/src/main/java/reactor/netty/http/server/IdleTimeoutHandler.java
index b8d333fcb2..d31f6310b8 100644
--- a/reactor-netty-http/src/main/java/reactor/netty/http/server/IdleTimeoutHandler.java
+++ b/reactor-netty-http/src/main/java/reactor/netty/http/server/IdleTimeoutHandler.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2023 VMware, Inc. or its affiliates, All Rights Reserved.
+ * Copyright (c) 2022-2024 VMware, Inc. or its affiliates, All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -53,7 +53,7 @@ protected void channelIdle(ChannelHandlerContext ctx, IdleStateEvent evt) {
 	}
 
 	static void addIdleTimeoutHandler(ChannelPipeline pipeline, @Nullable Duration idleTimeout) {
-		if (pipeline.get(NettyPipeline.IdleTimeoutHandler) == null && idleTimeout != null) {
+		if (idleTimeout != null && pipeline.get(NettyPipeline.IdleTimeoutHandler) == null) {
 			String baseName = null;
 			if (pipeline.get(NettyPipeline.HttpCodec) != null) {
 				baseName = NettyPipeline.HttpCodec;