feat[consumer]: support custom thread pool, providing the possibility…

… for different consumers to reuse thread pools
apache · Nov 28, 2024 · 48a755a · 48a755a
1 parent b638d4c
commit 48a755a
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 27 deletions.
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
@@ -16,6 +16,9 @@
  */
 package org.apache.rocketmq.client.consumer;
 
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.consumer.listener.MessageListener;
@@ -170,6 +173,19 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
      */
     private long adjustThreadPoolNumsThreshold = 100000;
 
+    /**
+     * Concurrently max span offset.it has no effect on sequential consumption
+     */
+    private ScheduledExecutorService consumeMessageScheduledExecutor;
+    /**
+     * Thread pool for handling expired messages
+     */
+    private ScheduledExecutorService cleanExpireMsgScheduledExecutor;
+    /**
+     * Thread pool for handling normal messages
+     */
+    private ExecutorService consumeExecutor;
+
     /**
      * Concurrently max span offset.it has no effect on sequential consumption
      */
@@ -927,6 +943,30 @@ public void setAdjustThreadPoolNumsThreshold(long adjustThreadPoolNumsThreshold)
         this.adjustThreadPoolNumsThreshold = adjustThreadPoolNumsThreshold;
     }
 
+    public ScheduledExecutorService getConsumeMessageScheduledExecutor() {
+        return consumeMessageScheduledExecutor;
+    }
+
+    public void setConsumeMessageScheduledExecutor(ScheduledExecutorService consumeMessageScheduledExecutor) {
+        this.consumeMessageScheduledExecutor = consumeMessageScheduledExecutor;
+    }
+
+    public ScheduledExecutorService getCleanExpireMsgScheduledExecutor() {
+        return cleanExpireMsgScheduledExecutor;
+    }
+
+    public void setCleanExpireMsgScheduledExecutor(ScheduledExecutorService cleanExpireMsgScheduledExecutor) {
+        this.cleanExpireMsgScheduledExecutor = cleanExpireMsgScheduledExecutor;
+    }
+
+    public ExecutorService getConsumeExecutor() {
+        return consumeExecutor;
+    }
+
+    public void setConsumeExecutor(ExecutorService consumeExecutor) {
+        this.consumeExecutor = consumeExecutor;
+    }
+
     public int getMaxReconsumeTimes() {
         return maxReconsumeTimes;
     }

diff --git a/...main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/...main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
@@ -54,7 +55,7 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService
     private final DefaultMQPushConsumer defaultMQPushConsumer;
     private final MessageListenerConcurrently messageListener;
     private final BlockingQueue<Runnable> consumeRequestQueue;
-    private final ThreadPoolExecutor consumeExecutor;
+    private final ExecutorService consumeExecutor;
     private final String consumerGroup;
 
     private final ScheduledExecutorService scheduledExecutorService;
@@ -68,18 +69,30 @@ public ConsumeMessageConcurrentlyService(DefaultMQPushConsumerImpl defaultMQPush
         this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer();
         this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup();
         this.consumeRequestQueue = new LinkedBlockingQueue<>();
-
         String consumerGroupTag = (consumerGroup.length() > 100 ? consumerGroup.substring(0, 100) : consumerGroup) + "_";
-        this.consumeExecutor = new ThreadPoolExecutor(
-            this.defaultMQPushConsumer.getConsumeThreadMin(),
-            this.defaultMQPushConsumer.getConsumeThreadMax(),
-            1000 * 60,
-            TimeUnit.MILLISECONDS,
-            this.consumeRequestQueue,
-            new ThreadFactoryImpl("ConsumeMessageThread_" + consumerGroupTag));
-
-        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_" + consumerGroupTag));
-        this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CleanExpireMsgScheduledThread_" + consumerGroupTag));
+        if (this.defaultMQPushConsumer.getConsumeExecutor() != null) {
+            this.consumeExecutor = this.defaultMQPushConsumer.getConsumeExecutor();
+        } else {
+            this.consumeExecutor = new ThreadPoolExecutor(
+                    this.defaultMQPushConsumer.getConsumeThreadMin(),
+                    this.defaultMQPushConsumer.getConsumeThreadMax(),
+                    1000 * 60,
+                    TimeUnit.MILLISECONDS,
+                    this.consumeRequestQueue,
+                    new ThreadFactoryImpl("ConsumeMessageThread_" + consumerGroupTag));
+        }
+
+        if (this.defaultMQPushConsumer.getConsumeMessageScheduledExecutor() != null) {
+            this.scheduledExecutorService = this.defaultMQPushConsumer.getConsumeMessageScheduledExecutor();
+        } else {
+            this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_" + consumerGroupTag));
+        }
+
+        if (this.defaultMQPushConsumer.getCleanExpireMsgScheduledExecutor() != null) {
+           this.cleanExpireMsgExecutors = this.defaultMQPushConsumer.getCleanExpireMsgScheduledExecutor();
+        } else {
+            this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CleanExpireMsgScheduledThread_" + consumerGroupTag));
+        }
     }
 
     public void start() {
@@ -108,7 +121,9 @@ public void updateCorePoolSize(int corePoolSize) {
         if (corePoolSize > 0
             && corePoolSize <= Short.MAX_VALUE
             && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) {
-            this.consumeExecutor.setCorePoolSize(corePoolSize);
+           if (consumeExecutor instanceof ThreadPoolExecutor) {
+               ((ThreadPoolExecutor)this.consumeExecutor).setCorePoolSize(corePoolSize);
+           }
         }
     }
 
@@ -124,7 +139,10 @@ public void decCorePoolSize() {
 
     @Override
     public int getCorePoolSize() {
-        return this.consumeExecutor.getCorePoolSize();
+        if (consumeExecutor instanceof ThreadPoolExecutor) {
+            return ((ThreadPoolExecutor)this.consumeExecutor).getCorePoolSize();
+        }
+        return -1;
     }
 
     @Override

diff --git a/.../src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/.../src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
@@ -58,7 +59,7 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService {
     private final DefaultMQPushConsumer defaultMQPushConsumer;
     private final MessageListenerOrderly messageListener;
     private final BlockingQueue<Runnable> consumeRequestQueue;
-    private final ThreadPoolExecutor consumeExecutor;
+    private final ExecutorService consumeExecutor;
     private final String consumerGroup;
     private final MessageQueueLock messageQueueLock = new MessageQueueLock();
     private final ScheduledExecutorService scheduledExecutorService;
@@ -72,17 +73,24 @@ public ConsumeMessageOrderlyService(DefaultMQPushConsumerImpl defaultMQPushConsu
         this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer();
         this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup();
         this.consumeRequestQueue = new LinkedBlockingQueue<>();
-
         String consumerGroupTag = (consumerGroup.length() > 100 ? consumerGroup.substring(0, 100) : consumerGroup) + "_";
-        this.consumeExecutor = new ThreadPoolExecutor(
-            this.defaultMQPushConsumer.getConsumeThreadMin(),
-            this.defaultMQPushConsumer.getConsumeThreadMax(),
-            1000 * 60,
-            TimeUnit.MILLISECONDS,
-            this.consumeRequestQueue,
-            new ThreadFactoryImpl("ConsumeMessageThread_" + consumerGroupTag));
-
-        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_" + consumerGroupTag));
+        if (this.defaultMQPushConsumer.getConsumeExecutor() != null) {
+            this.consumeExecutor = this.defaultMQPushConsumer.getConsumeExecutor();
+        } else {
+            this.consumeExecutor = new ThreadPoolExecutor(
+                    this.defaultMQPushConsumer.getConsumeThreadMin(),
+                    this.defaultMQPushConsumer.getConsumeThreadMax(),
+                    1000 * 60,
+                    TimeUnit.MILLISECONDS,
+                    this.consumeRequestQueue,
+                    new ThreadFactoryImpl("ConsumeMessageThread_" + consumerGroupTag));
+        }
+
+        if (this.defaultMQPushConsumer.getConsumeMessageScheduledExecutor() != null) {
+            this.scheduledExecutorService = this.defaultMQPushConsumer.getConsumeMessageScheduledExecutor();
+        } else {
+            this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_" + consumerGroupTag));
+        }
     }
 
     @Override
@@ -120,8 +128,9 @@ public void updateCorePoolSize(int corePoolSize) {
         if (corePoolSize > 0
             && corePoolSize <= Short.MAX_VALUE
             && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) {
-            this.consumeExecutor.setCorePoolSize(corePoolSize);
-        }
+            if (consumeExecutor instanceof ThreadPoolExecutor) {
+                ((ThreadPoolExecutor)this.consumeExecutor).setCorePoolSize(corePoolSize);
+            }        }
     }
 
     @Override