關鍵詞：Java，多執行緒，訊息佇列，rocketmq
多執行緒一個用例之一就是訊息的快速消費，比如我們有一個訊息佇列我們希望以更快的速度消費訊息，假如我們用的是rocketmq，我們從中獲取訊息，然後使用多執行緒處理。
這是一個抽象類，針對不同的任務可能有不同的處理邏輯，對於不同的任務去繼承這個CrazyTask 實現他的process方法。
package crazyConsumer;
import com.google.common.collect.Lists;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
/**
* {@code @author:} keboom
* {@code @date:} 2023/11/17
* {@code @description:}
*/
public abstract class CrazyTask {
String taskName;
int threadNum;
volatile boolean isTerminated;
// every partition data num.
// for example: I receive 5 messages, partitionDataNum is 2, then i will partition 5 messages to 3 parts, 2,2,1
int partitionDataCount = 2;
abstract void process(Message message);
void doExecute(SimpleConsumer consumer) {
while (true) {
// 此消費者每次主動拉取訊息佇列中訊息
List<Message> messages = consumer.receive();
if (messages.isEmpty()) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
continue;
}
// 獲取處理此topic或者說處理此型別task的執行緒池
ExecutorService executor = CrazyTaskUtil.getOrInitExecutor(taskName, threadNum);
// 將訊息分片，每個執行緒處理一部分訊息
List<List<Message>> partition = Lists.partition(messages, partitionDataCount);
// 以訊息分片數初始化CountDownLatch，每個執行緒處理完一片訊息，countDown一次
// 當countDownLatch為0時，說明所有訊息都處理完了，countDownLatch.await();繼續向下執行
CountDownLatch countDownLatch = new CountDownLatch(partition.size());
partition.forEach(messageList -> {
executor.execute(() -> {
messageList.forEach(message -> {
process(message);
consumer.ack(message);
});
countDownLatch.countDown();
});
});
try {
countDownLatch.await();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (isTerminated) {
break;
}
}
// 釋放執行緒池
CrazyTaskUtil.shutdownThreadPool(taskName);
}
void terminate() {
isTerminated = true;
System.out.println();
System.out.println(taskName + " shut down");
}
public String getTaskName() {
return taskName;
}
}
package crazyConsumer;
/**
* {@code @author:} keboom
* {@code @date:} 2023/11/17
* {@code @description:}
*/
public class PhoneTask extends CrazyTask {
public PhoneTask(String taskName, int threadNum) {
this.taskName = taskName;
// default thread num
this.threadNum = threadNum;
this.isTerminated = false;
}
@Override
void process(Message message) {
System.out.println(Thread.currentThread().getName() +" process "+ message.toString());
try {
Thread.sleep(30);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
@Override
public String toString() {
return "PhoneTask{" +
"taskName='" + taskName + '\'' +
", threadNum=" + threadNum +
", isTerminated=" + isTerminated +
'}';
}
}
package crazyConsumer;
/**
* {@code @author:} keboom
* {@code @date:} 2023/11/17
* {@code @description:}
*/
public class EmailTask extends CrazyTask{
public EmailTask(String taskName, int threadNum) {
this.taskName = taskName;
// default thread num
this.threadNum = threadNum;
this.isTerminated = false;
}
@Override
void process(Message message) {
// do something
System.out.println(Thread.currentThread().getName() +" process "+ message.toString());
try {
Thread.sleep(20);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
@Override
public String toString() {
return "EmailTask{" +
"taskName='" + taskName + '\'' +
", threadNum=" + threadNum +
", isTerminated=" + isTerminated +
'}';
}
}
建立銷燬執行緒池的工具類
package crazyConsumer;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.Map;
import java.util.concurrent.*;
/**
* {@code @author:} keboom
* {@code @date:} 2023/11/17
* {@code @description:}
*/
public class CrazyTaskUtil {
private static final Map<String, ExecutorService> executors = new ConcurrentHashMap<>();
public static ExecutorService getOrInitExecutor(String taskName, int threadNum) {
ExecutorService executorService = executors.get(taskName);
if (executorService == null) {
synchronized (CrazyTaskUtil.class) {
executorService = executors.get(taskName);
if (executorService == null) {
executorService = initPool(taskName, threadNum);
executors.put(taskName, executorService);
}
}
}
return executorService;
}
private static ExecutorService initPool(String taskName, int threadNum) {
// init pool
return new ThreadPoolExecutor(threadNum, threadNum,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(),
new ThreadFactoryBuilder().setNameFormat(taskName + "-%d").build());
}
public static void shutdownThreadPool(String taskName) {
ExecutorService remove = executors.remove(taskName);
if (remove != null) {
remove.shutdown();
}
}
}
程式入口
package crazyConsumer;
import java.util.ArrayList;
/**
* {@code @author:} keboom
* {@code @date:} 2023/11/17
* {@code @description:}
*/
public class Main {
/**
* 一種多執行緒消費場景。比如我們有一個消費佇列，裡面有各種訊息，我們需要儘快的消費他們，不同的訊息對應不同的業務
*
* @param args
*/
public static void main(String[] args) throws InterruptedException {
// 比方說我們這個有rocketmq不同主題的consumer
/*
List<MessageView> messageViewList = null;
try {
messageViewList = simpleConsumer.receive(10, Duration.ofSeconds(30));
messageViewList.forEach(messageView -> {
System.out.println(messageView);
//消費處理完成後，需要主動呼叫ACK提交消費結果。
try {
simpleConsumer.ack(messageView);
} catch (ClientException e) {
e.printStackTrace();
}
});
} catch (ClientException e) {
//如果遇到系統流控等原因造成拉取失敗，需要重新發起獲取訊息請求。
e.printStackTrace();
}
*/
// 想要實現多執行緒消費訊息，我們希望有一個任務，此任務能夠不停的拉取訊息，然後建立子執行緒池去消費訊息。
// 停止任務後，需要將任務中的訊息消費完後，再關閉任務。
ArrayList<CrazyTask> tasks = new ArrayList<>();
tasks.add(new PhoneTask("phoneTask", 2));
tasks.add(new EmailTask("emailTask", 3));
for (CrazyTask task : tasks) {
new Thread(() -> {
task.doExecute(new SimpleConsumer("topic"+task.getTaskName().charAt(0), "tagA"));
}).start();
}
// task running
Thread.sleep(150);
// task terminated
tasks.forEach(CrazyTask::terminate);
}
}
receive message: [Message{messageBody='topice-tagA-0-1700470193487'}, Message{messageBody='topice-tagA-1-1700470193487'}, Message{messageBody='topice-tagA-2-1700470193487'}, Message{messageBody='topice-tagA-3-1700470193487'}, Message{messageBody='topice-tagA-4-1700470193487'}]
receive message: [Message{messageBody='topicp-tagA-0-1700470193487'}, Message{messageBody='topicp-tagA-1-1700470193487'}, Message{messageBody='topicp-tagA-2-1700470193487'}, Message{messageBody='topicp-tagA-3-1700470193487'}, Message{messageBody='topicp-tagA-4-1700470193487'}]
phoneTask-0 process Message{messageBody='topicp-tagA-0-1700470193487'}
emailTask-1 process Message{messageBody='topice-tagA-2-1700470193487'}
emailTask-0 process Message{messageBody='topice-tagA-0-1700470193487'}
phoneTask-1 process Message{messageBody='topicp-tagA-2-1700470193487'}
emailTask-2 process Message{messageBody='topice-tagA-4-1700470193487'}
ack message: Message{messageBody='topice-tagA-2-1700470193487'}
emailTask-1 process Message{messageBody='topice-tagA-3-1700470193487'}
ack message: Message{messageBody='topice-tagA-4-1700470193487'}
ack message: Message{messageBody='topice-tagA-0-1700470193487'}
emailTask-0 process Message{messageBody='topice-tagA-1-1700470193487'}
ack message: Message{messageBody='topicp-tagA-2-1700470193487'}
ack message: Message{messageBody='topicp-tagA-0-1700470193487'}
phoneTask-0 process Message{messageBody='topicp-tagA-1-1700470193487'}
phoneTask-1 process Message{messageBody='topicp-tagA-3-1700470193487'}
ack message: Message{messageBody='topice-tagA-1-1700470193487'}
ack message: Message{messageBody='topice-tagA-3-1700470193487'}
receive message: [Message{messageBody='topice-tagA-0-1700470193570'}, Message{messageBody='topice-tagA-1-1700470193570'}, Message{messageBody='topice-tagA-2-1700470193570'}, Message{messageBody='topice-tagA-3-1700470193570'}, Message{messageBody='topice-tagA-4-1700470193570'}]
emailTask-0 process Message{messageBody='topice-tagA-2-1700470193570'}
emailTask-2 process Message{messageBody='topice-tagA-0-1700470193570'}
emailTask-1 process Message{messageBody='topice-tagA-4-1700470193570'}
ack message: Message{messageBody='topicp-tagA-3-1700470193487'}
ack message: Message{messageBody='topicp-tagA-1-1700470193487'}
phoneTask-1 process Message{messageBody='topicp-tagA-4-1700470193487'}
ack message: Message{messageBody='topice-tagA-0-1700470193570'}
ack message: Message{messageBody='topice-tagA-4-1700470193570'}
ack message: Message{messageBody='topice-tagA-2-1700470193570'}
emailTask-0 process Message{messageBody='topice-tagA-3-1700470193570'}
emailTask-2 process Message{messageBody='topice-tagA-1-1700470193570'}
ack message: Message{messageBody='topicp-tagA-4-1700470193487'}
receive message: [Message{messageBody='topicp-tagA-0-1700470193618'}, Message{messageBody='topicp-tagA-1-1700470193618'}, Message{messageBody='topicp-tagA-2-1700470193618'}, Message{messageBody='topicp-tagA-3-1700470193618'}, Message{messageBody='topicp-tagA-4-1700470193618'}]
phoneTask-0 process Message{messageBody='topicp-tagA-0-1700470193618'}
phoneTask-1 process Message{messageBody='topicp-tagA-2-1700470193618'}
ack message: Message{messageBody='topice-tagA-1-1700470193570'}
ack message: Message{messageBody='topice-tagA-3-1700470193570'}
receive message: [Message{messageBody='topice-tagA-0-1700470193634'}, Message{messageBody='topice-tagA-1-1700470193634'}, Message{messageBody='topice-tagA-2-1700470193634'}, Message{messageBody='topice-tagA-3-1700470193634'}, Message{messageBody='topice-tagA-4-1700470193634'}]
emailTask-1 process Message{messageBody='topice-tagA-0-1700470193634'}
emailTask-0 process Message{messageBody='topice-tagA-4-1700470193634'}
emailTask-2 process Message{messageBody='topice-tagA-2-1700470193634'}
ack message: Message{messageBody='topicp-tagA-2-1700470193618'}
ack message: Message{messageBody='topicp-tagA-0-1700470193618'}
phoneTask-0 process Message{messageBody='topicp-tagA-1-1700470193618'}
phoneTask-1 process Message{messageBody='topicp-tagA-3-1700470193618'}
phoneTask shut down
emailTask shut down
ack message: Message{messageBody='topice-tagA-0-1700470193634'}
ack message: Message{messageBody='topice-tagA-2-1700470193634'}
emailTask-1 process Message{messageBody='topice-tagA-1-1700470193634'}
ack message: Message{messageBody='topice-tagA-4-1700470193634'}
emailTask-2 process Message{messageBody='topice-tagA-3-1700470193634'}
ack message: Message{messageBody='topicp-tagA-3-1700470193618'}
ack message: Message{messageBody='topicp-tagA-1-1700470193618'}
phoneTask-1 process Message{messageBody='topicp-tagA-4-1700470193618'}
ack message: Message{messageBody='topice-tagA-3-1700470193634'}
ack message: Message{messageBody='topice-tagA-1-1700470193634'}
ack message: Message{messageBody='topicp-tagA-4-1700470193618'}
可以看到結果是，當每次收到的訊息消費完後會拉取新的訊息。當執行shutdown任務時，會將當前任務執行完後再銷燬執行緒池。