study/Spring-Cloud-Alibaba
1
0
Fork 0
mirror of https://gitee.com/mirrors/Spring-Cloud-Alibaba.git synced 2021-06-26 13:25:11 +08:00
Code Issues Releases Wiki Activity

Polish #541, enhance concurrent handle

Browse Source
This commit is contained in:
fangjian0423 2019-04-16 10:35:32 +08:00
parent ad944513b7
commit d37f21885b
2 changed files with 99 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/RocketMQMessageQueueChooser.java
View File

@ -27,7 +27,7 @@ import org.apache.rocketmq.common.message.MessageQueue;
*/ */
public class RocketMQMessageQueueChooser { public class RocketMQMessageQueueChooser {
private int queueIndex = 0; private volatile int queueIndex = 0;
private volatile List<MessageQueue> messageQueues; private volatile List<MessageQueue> messageQueues;
@ -35,7 +35,7 @@ public class RocketMQMessageQueueChooser {
return messageQueues.get(queueIndex); return messageQueues.get(queueIndex);
} }
public synchronized int requeue() { public int requeue() {
if (queueIndex - 1 < 0) { if (queueIndex - 1 < 0) {
this.queueIndex = messageQueues.size() - 1; this.queueIndex = messageQueues.size() - 1;
} }

203
spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQMessageSource.java
View File

@ -18,8 +18,6 @@ package org.springframework.cloud.stream.binder.rocketmq.integration;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.MessageQueueListener;
@ -67,7 +65,7 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
private final String group; private final String group;
private final Lock lock = new ReentrantLock(); private final Object consumerMonitor = new Object();
private DefaultMQPullConsumer consumer; private DefaultMQPullConsumer consumer;
@ -97,7 +95,7 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
} }
@Override @Override
public void start() { public synchronized void start() {
if (this.isRunning()) { if (this.isRunning()) {
throw new IllegalStateException( throw new IllegalStateException(
"pull consumer already running. " + this.toString()); "pull consumer already running. " + this.toString());
@ -151,7 +149,7 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
} }
@Override @Override
public void stop() { public synchronized void stop() {
if (this.isRunning()) { if (this.isRunning()) {
this.setRunning(false); this.setRunning(false);
consumer.shutdown(); consumer.shutdown();
@ -159,73 +157,64 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
} }
@Override @Override
public boolean isRunning() { public synchronized boolean isRunning() {
return running; return running;
} }
@Override @Override
protected Object doReceive() { protected synchronized Object doReceive() {
if (messageQueueChooser.getMessageQueues() == null if (messageQueueChooser.getMessageQueues() == null
|| messageQueueChooser.getMessageQueues().size() == 0) { || messageQueueChooser.getMessageQueues().size() == 0) {
return null; return null;
} }
if (lock.tryLock()) { try {
try { int count = 0;
int count = 0; while (count < messageQueueChooser.getMessageQueues().size()) {
while (count < messageQueueChooser.getMessageQueues().size()) { MessageQueue messageQueue;
synchronized (this.consumerMonitor) {
MessageQueue messageQueue = messageQueueChooser.choose(); messageQueue = messageQueueChooser.choose();
long offset = consumer.fetchConsumeOffset(messageQueue,
rocketMQConsumerProperties.getExtension().isFromStore());
log.debug(
"topic='{}', group='{}', messageQueue='{}', offset now='{}'",
this.topic, this.group, messageQueue, offset);
PullResult pullResult;
if (messageSelector != null) {
pullResult = consumer.pull(messageQueue, messageSelector, offset,
1);
}
else {
pullResult = consumer.pull(messageQueue, (String) null, offset,
1);
}
if (pullResult.getPullStatus() == PullStatus.FOUND) {
List<MessageExt> messageExtList = pullResult.getMsgFoundList();
Message message = RocketMQUtil
.convertToSpringMessage(messageExtList.get(0));
AcknowledgmentCallback ackCallback = this.ackCallbackFactory
.createCallback(
new RocketMQAckInfo(messageQueue, pullResult,
consumer, messageQueueChooser, offset));
Message messageResult = MessageBuilder.fromMessage(message)
.setHeader(
IntegrationMessageHeaderAccessor.ACKNOWLEDGMENT_CALLBACK,
ackCallback)
.build();
messageQueueChooser.increment();
return messageResult;
}
else {
log.debug("messageQueue='{}' PullResult='{}' with topic `{}`",
messageQueueChooser.getMessageQueues(),
pullResult.getPullStatus(), topic);
}
messageQueueChooser.increment(); messageQueueChooser.increment();
count++;
} }
long offset = consumer.fetchConsumeOffset(messageQueue,
rocketMQConsumerProperties.getExtension().isFromStore());
log.debug("topic='{}', group='{}', messageQueue='{}', offset now='{}'",
this.topic, this.group, messageQueue, offset);
PullResult pullResult;
if (messageSelector != null) {
pullResult = consumer.pull(messageQueue, messageSelector, offset, 1);
}
else {
pullResult = consumer.pull(messageQueue, (String) null, offset, 1);
}
if (pullResult.getPullStatus() == PullStatus.FOUND) {
List<MessageExt> messageExtList = pullResult.getMsgFoundList();
Message message = RocketMQUtil
.convertToSpringMessage(messageExtList.get(0));
AcknowledgmentCallback ackCallback = this.ackCallbackFactory
.createCallback(new RocketMQAckInfo(messageQueue, pullResult,
consumer, offset));
Message messageResult = MessageBuilder.fromMessage(message).setHeader(
IntegrationMessageHeaderAccessor.ACKNOWLEDGMENT_CALLBACK,
ackCallback).build();
return messageResult;
}
else {
log.debug("messageQueue='{}' PullResult='{}' with topic `{}`",
messageQueueChooser.getMessageQueues(),
pullResult.getPullStatus(), topic);
}
count++;
} }
catch (Exception e) { }
log.error("Consumer pull error: " + e.getMessage(), e); catch (Exception e) {
} log.error("Consumer pull error: " + e.getMessage(), e);
finally {
lock.unlock();
}
} }
return null; return null;
} }
@ -235,20 +224,15 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
return "rocketmq:message-source"; return "rocketmq:message-source";
} }
public void setRunning(boolean running) { public synchronized void setRunning(boolean running) {
this.running = running; this.running = running;
} }
public void resetMessageQueues(Set<MessageQueue> queueSet) { public synchronized void resetMessageQueues(Set<MessageQueue> queueSet) {
lock.lock(); log.info("resetMessageQueues, topic='{}', messageQueue=`{}`", topic, queueSet);
try { synchronized (this.consumerMonitor) {
log.info("resetMessageQueues, topic='{}', messageQueue=`{}`", topic,
queueSet);
this.messageQueueChooser.reset(queueSet); this.messageQueueChooser.reset(queueSet);
} }
finally {
lock.unlock();
}
} }
public static class RocketMQCallbackFactory public static class RocketMQCallbackFactory
@ -295,35 +279,42 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
@Override @Override
public void acknowledge(Status status) { public void acknowledge(Status status) {
Assert.notNull(status, "'status' cannot be null"); Assert.notNull(status, "'status' cannot be null");
if (this.acknowledged) {
throw new IllegalStateException("Already acknowledged");
}
log.debug("acknowledge(" + status.name() + ") for " + this); log.debug("acknowledge(" + status.name() + ") for " + this);
try { synchronized (this.ackInfo.getConsumerMonitor()) {
switch (status) { try {
case ACCEPT: switch (status) {
case REJECT: case ACCEPT:
ackInfo.getConsumer().updateConsumeOffset(ackInfo.getMessageQueue(), case REJECT:
ackInfo.getPullResult().getNextBeginOffset()); ackInfo.getConsumer().updateConsumeOffset(
log.debug("messageQueue='{}' offset update to `{}`", ackInfo.getMessageQueue(),
ackInfo.getMessageQueue(), ackInfo.getPullResult().getNextBeginOffset());
String.valueOf(ackInfo.getPullResult().getNextBeginOffset())); log.debug("messageQueue='{}' offset update to `{}`",
break; ackInfo.getMessageQueue(), String.valueOf(
case REQUEUE: ackInfo.getPullResult().getNextBeginOffset()));
// decrease index and update offset of messageQueue of ackInfo break;
int oldIndex = ackInfo.getMessageQueueChooser().requeue(); case REQUEUE:
ackInfo.getConsumer().updateConsumeOffset(ackInfo.getMessageQueue(), // decrease index and update offset of messageQueue of ackInfo
ackInfo.getOldOffset()); int oldIndex = ackInfo.getMessageQueueChooser().requeue();
log.debug( ackInfo.getConsumer().updateConsumeOffset(
"messageQueue='{}' offset requeue to index:`{}`, oldOffset:'{}'", ackInfo.getMessageQueue(), ackInfo.getOldOffset());
ackInfo.getMessageQueue(), oldIndex, ackInfo.getOldOffset()); log.debug(
break; "messageQueue='{}' offset requeue to index:`{}`, oldOffset:'{}'",
default: ackInfo.getMessageQueue(), oldIndex,
break; ackInfo.getOldOffset());
break;
default:
break;
}
}
catch (MQClientException e) {
log.error("acknowledge error: " + e.getErrorMessage(), e);
}
finally {
this.acknowledged = true;
} }
}
catch (MQClientException e) {
log.error("acknowledge error: " + e.getErrorMessage(), e);
}
finally {
this.acknowledged = true;
} }
} }
@ -334,7 +325,7 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
} }
} }
public static class RocketMQAckInfo { public class RocketMQAckInfo {
private final MessageQueue messageQueue; private final MessageQueue messageQueue;
@ -342,17 +333,13 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
private final DefaultMQPullConsumer consumer; private final DefaultMQPullConsumer consumer;
private final RocketMQMessageQueueChooser messageQueueChooser;
private final long oldOffset; private final long oldOffset;
public RocketMQAckInfo(MessageQueue messageQueue, PullResult pullResult, public RocketMQAckInfo(MessageQueue messageQueue, PullResult pullResult,
DefaultMQPullConsumer consumer, DefaultMQPullConsumer consumer, long oldOffset) {
RocketMQMessageQueueChooser messageQueueChooser, long oldOffset) {
this.messageQueue = messageQueue; this.messageQueue = messageQueue;
this.pullResult = pullResult; this.pullResult = pullResult;
this.consumer = consumer; this.consumer = consumer;
this.messageQueueChooser = messageQueueChooser;
this.oldOffset = oldOffset; this.oldOffset = oldOffset;
} }
@ -369,18 +356,22 @@ public class RocketMQMessageSource extends AbstractMessageSource<Object>
} }
public RocketMQMessageQueueChooser getMessageQueueChooser() { public RocketMQMessageQueueChooser getMessageQueueChooser() {
return messageQueueChooser; return RocketMQMessageSource.this.messageQueueChooser;
} }
public long getOldOffset() { public long getOldOffset() {
return oldOffset; return oldOffset;
} }
public Object getConsumerMonitor() {
return RocketMQMessageSource.this.consumerMonitor;
}
@Override @Override
public String toString() { public String toString() {
return "RocketMQAckInfo{" + "messageQueue=" + messageQueue + ", pullResult=" return "RocketMQAckInfo{" + "messageQueue=" + messageQueue + ", pullResult="
+ pullResult + ", consumer=" + consumer + ", messageQueueChooser=" + pullResult + ", consumer=" + consumer + ", oldOffset=" + oldOffset
+ messageQueueChooser + ", oldOffset=" + oldOffset + '}'; + '}';
} }
} }