Skip to content

[ISSUE #10936] Avoid temporary buffers during batch message encoding #10938

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ai-yang wants to merge 2 commits into
base: develop
Choose a base branch
from
+91 −21
Open

[ISSUE #10936] Avoid temporary buffers during batch message encoding #10938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9fe5950
[ISSUE #10936] Avoid temporary buffers during batch message encoding
ai-yang Aug 15, 2026
d66b4f8
[ISSUE #10936] Use Unicode escapes in encoding test
ai-yang Aug 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 33 additions & 21 deletions common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,20 +661,29 @@ public static Map<String, String> string2messageProperties(final String properti
}

public static byte[] encodeMessage(Message message) {
//only need flag, body, properties
byte[] body = message.getBody();
int bodyLen = body.length;
String properties = messageProperties2String(message.getProperties());
byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);
//note properties length must not more than Short.MAX
int storeSize = encodedMessageSize(body, propertiesBytes);
ByteBuffer byteBuffer = ByteBuffer.allocate(storeSize);
encodeMessage(message, body, propertiesBytes, storeSize, byteBuffer);
return byteBuffer.array();
}

private static int encodedMessageSize(byte[] body, byte[] propertiesBytes) {
// Note properties length must not be greater than Short.MAX_VALUE.
short propertiesLength = (short) propertiesBytes.length;
int storeSize = 4 // 1 TOTALSIZE
return 4 // 1 TOTALSIZE
+ 4 // 2 MAGICCOD
+ 4 // 3 BODYCRC
+ 4 // 4 FLAG
+ 4 + bodyLen // 4 BODY
+ 4 + body.length // 5 BODY
+ 2 + propertiesLength;
ByteBuffer byteBuffer = ByteBuffer.allocate(storeSize);
}

private static void encodeMessage(Message message, byte[] body, byte[] propertiesBytes, int storeSize,
ByteBuffer byteBuffer) {
short propertiesLength = (short) propertiesBytes.length;
// 1 TOTALSIZE
byteBuffer.putInt(storeSize);

Expand All @@ -689,14 +698,12 @@ public static byte[] encodeMessage(Message message) {
byteBuffer.putInt(flag);

// 5 BODY
byteBuffer.putInt(bodyLen);
byteBuffer.putInt(body.length);
byteBuffer.put(body);

// 6 properties
byteBuffer.putShort(propertiesLength);
byteBuffer.put(propertiesBytes);

return byteBuffer.array();
}

public static Message decodeMessage(ByteBuffer byteBuffer) throws Exception {
Expand Down Expand Up @@ -731,21 +738,26 @@ public static Message decodeMessage(ByteBuffer byteBuffer) throws Exception {
}

public static byte[] encodeMessages(List<Message> messages) {
//TO DO refactor, accumulate in one buffer, avoid copies
List<byte[]> encodedMessages = new ArrayList<>(messages.size());
int allSize = 0;
byte[][] bodies = new byte[messages.size()][];
byte[][] propertiesBytes = new byte[messages.size()][];
int totalSize = 0;
int index = 0;
for (Message message : messages) {
byte[] tmp = encodeMessage(message);
encodedMessages.add(tmp);
allSize += tmp.length;
byte[] body = message.getBody();
byte[] encodedProperties = messageProperties2String(message.getProperties()).getBytes(CHARSET_UTF8);
bodies[index] = body;
propertiesBytes[index++] = encodedProperties;
totalSize += encodedMessageSize(body, encodedProperties);
}
byte[] allBytes = new byte[allSize];
int pos = 0;
for (byte[] bytes : encodedMessages) {
System.arraycopy(bytes, 0, allBytes, pos, bytes.length);
pos += bytes.length;

ByteBuffer byteBuffer = ByteBuffer.allocate(totalSize);
index = 0;
for (Message message : messages) {
byte[] body = bodies[index];
byte[] encodedProperties = propertiesBytes[index++];
encodeMessage(message, body, encodedProperties, encodedMessageSize(body, encodedProperties), byteBuffer);
}
return allBytes;
return byteBuffer.array();
}

public static List<Message> decodeMessages(ByteBuffer byteBuffer) throws Exception {
Expand Down
58 changes: 58 additions & 0 deletions common/src/test/java/org/apache/rocketmq/common/MessageEncodeDecodeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.rocketmq.common.message.MessageDecoder;
import org.junit.Test;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertTrue;

public class MessageEncodeDecodeTest {
Expand Down Expand Up @@ -72,4 +73,61 @@ public void testEncodeDecodeList() throws Exception {

}
}

@Test
public void testMessageEncodingWireFormatGoldenBytes() {
Message message = new Message();
message.setBody(new byte[] {0x01, 0x02});
message.setFlag(0x01020304);

byte[] encodedProperties = new byte[0];
int storeSize = 4 + 4 + 4 + 4 + 4 + message.getBody().length + 2 + encodedProperties.length;
ByteBuffer expected = ByteBuffer.allocate(storeSize);
expected.putInt(storeSize);
expected.putInt(0);
expected.putInt(0);
expected.putInt(message.getFlag());
expected.putInt(message.getBody().length);
expected.put(message.getBody());
expected.putShort((short) encodedProperties.length);
expected.put(encodedProperties);

assertArrayEquals(expected.array(), MessageDecoder.encodeMessage(message));
assertArrayEquals(expected.array(), MessageDecoder.encodeMessages(Arrays.asList(message)));
}

@Test
public void testBatchEncodingMatchesConcatenatedSingleMessages() {
Message emptyMessage = new Message();
emptyMessage.setBody(new byte[0]);
emptyMessage.setFlag(-1);

Message unicodeMessage = new Message("topic", "payload".getBytes(MessageDecoder.CHARSET_UTF8));
unicodeMessage.setFlag(7);
unicodeMessage.putUserProperty("region", "\u534e\u4e1c");

byte[] largeBody = new byte[4096];
Arrays.fill(largeBody, (byte) 0x5A);
Message largeMessage = new Message("topic", largeBody);
largeMessage.setFlag(Integer.MAX_VALUE);
largeMessage.putUserProperty("key", "value");
largeMessage.putUserProperty("trace", "enabled");

List<Message> messages = Arrays.asList(emptyMessage, unicodeMessage, largeMessage);
List<byte[]> individuallyEncoded = new ArrayList<>(messages.size());
int expectedLength = 0;
for (Message message : messages) {
byte[] encoded = MessageDecoder.encodeMessage(message);
individuallyEncoded.add(encoded);
expectedLength += encoded.length;
}

ByteBuffer expected = ByteBuffer.allocate(expectedLength);
for (byte[] encoded : individuallyEncoded) {
expected.put(encoded);
}

assertArrayEquals(expected.array(), MessageDecoder.encodeMessages(messages));
assertArrayEquals(new byte[0], MessageDecoder.encodeMessages(new ArrayList<Message>()));
}
}
Loading