Skip to content

Commit c354fa4

Browse files
authored
HttpPostRequestDecoder: retain instead of copy when first buf is last (netty#10209)
Motivations ----------- There is no need to copy the "offered" ByteBuf in HttpPostRequestDecoder when the first HttpContent ByteBuf is also the last (LastHttpContent) as the full content can immediately be decoded. No extra bookeeping needed. Modifications ------------- HttpPostMultipartRequestDecoder - Retain the first ByteBuf when it is both the first HttpContent offered to the decoder and is also LastHttpContent. - Retain slices of the final buffers values Results ------- ByteBufs of FullHttpMessage decoded by HttpPostRequestDecoder are no longer unnecessarily copied. Attributes are extracted as retained slices when the content is multi-part. Non-multi-part content continues to return Unpooled buffers. Partially addresses issue netty#10200
1 parent 9751bb3 commit c354fa4

3 files changed

Lines changed: 351 additions & 155 deletions

File tree

‎codec-http/src/main/java/io/netty/handler/codec/http/multipart/HttpPostMultipartRequestDecoder.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -321,18 +321,23 @@ public InterfaceHttpData getBodyHttpData(String name) {
321321
public HttpPostMultipartRequestDecoder offer(HttpContent content) {
322322
checkDestroyed();
323323

324-
// Maybe we should better not copy here for performance reasons but this will need
325-
// more care by the caller to release the content in a correct manner later
326-
// So maybe something to optimize on a later stage
324+
if (content instanceof LastHttpContent) {
325+
isLastChunk = true;
326+
}
327+
327328
ByteBuf buf = content.content();
328329
if (undecodedChunk == null) {
329-
undecodedChunk = buf.copy();
330+
undecodedChunk = isLastChunk
331+
// Take a slice instead of copying when the first chunk is also the last
332+
// as undecodedChunk.writeBytes will never be called.
333+
? buf.retainedSlice()
334+
// Maybe we should better not copy here for performance reasons but this will need
335+
// more care by the caller to release the content in a correct manner later
336+
// So maybe something to optimize on a later stage
337+
: buf.copy();
330338
} else {
331339
undecodedChunk.writeBytes(buf);
332340
}
333-
if (content instanceof LastHttpContent) {
334-
isLastChunk = true;
335-
}
336341
parseBody();
337342
if (undecodedChunk != null && undecodedChunk.writerIndex() > discardThreshold) {
338343
undecodedChunk.discardReadBytes();
@@ -1315,7 +1320,7 @@ private static boolean loadDataMultipartStandard(ByteBuf undecodedChunk, String
13151320
if (prevByte == HttpConstants.CR) {
13161321
lastPosition--;
13171322
}
1318-
ByteBuf content = undecodedChunk.copy(startReaderIndex, lastPosition - startReaderIndex);
1323+
ByteBuf content = undecodedChunk.retainedSlice(startReaderIndex, lastPosition - startReaderIndex);
13191324
try {
13201325
httpData.addContent(content, delimiterFound);
13211326
} catch (IOException e) {
@@ -1364,7 +1369,7 @@ private static boolean loadDataMultipart(ByteBuf undecodedChunk, String delimite
13641369
lastRealPos--;
13651370
}
13661371
final int lastPosition = sao.getReadPosition(lastRealPos);
1367-
final ByteBuf content = undecodedChunk.copy(startReaderIndex, lastPosition - startReaderIndex);
1372+
final ByteBuf content = undecodedChunk.retainedSlice(startReaderIndex, lastPosition - startReaderIndex);
13681373
try {
13691374
httpData.addContent(content, delimiterFound);
13701375
} catch (IOException e) {

‎codec-http/src/main/java/io/netty/handler/codec/http/multipart/HttpPostStandardRequestDecoder.java‎

Lines changed: 86 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@
2626
import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder.ErrorDataDecoderException;
2727
import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder.MultiPartStatus;
2828
import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder.NotEnoughDataDecoderException;
29+
import io.netty.util.ByteProcessor;
30+
import io.netty.util.CharsetUtil;
2931
import io.netty.util.internal.PlatformDependent;
32+
import io.netty.util.internal.StringUtil;
3033

3134
import java.io.IOException;
3235
import java.nio.charset.Charset;
@@ -280,18 +283,23 @@ public InterfaceHttpData getBodyHttpData(String name) {
280283
public HttpPostStandardRequestDecoder offer(HttpContent content) {
281284
checkDestroyed();
282285

283-
// Maybe we should better not copy here for performance reasons but this will need
284-
// more care by the caller to release the content in a correct manner later
285-
// So maybe something to optimize on a later stage
286+
if (content instanceof LastHttpContent) {
287+
isLastChunk = true;
288+
}
289+
286290
ByteBuf buf = content.content();
287291
if (undecodedChunk == null) {
288-
undecodedChunk = buf.copy();
292+
undecodedChunk = isLastChunk
293+
// Take a slice instead of copying when the first chunk is also the last
294+
// as undecodedChunk.writeBytes will never be called.
295+
? buf.retainedSlice()
296+
// Maybe we should better not copy here for performance reasons but this will need
297+
// more care by the caller to release the content in a correct manner later
298+
// So maybe something to optimize on a later stage
299+
: buf.copy();
289300
} else {
290301
undecodedChunk.writeBytes(buf);
291302
}
292-
if (content instanceof LastHttpContent) {
293-
isLastChunk = true;
294-
}
295303
parseBody();
296304
if (undecodedChunk != null && undecodedChunk.writerIndex() > discardThreshold) {
297305
undecodedChunk.discardReadBytes();
@@ -429,7 +437,7 @@ private void parseBodyAttributesStandard() {
429437
if (read == '&') {
430438
currentStatus = MultiPartStatus.DISPOSITION;
431439
ampersandpos = currentpos - 1;
432-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
440+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
433441
firstpos = currentpos;
434442
contRead = true;
435443
} else if (read == HttpConstants.CR) {
@@ -439,7 +447,7 @@ private void parseBodyAttributesStandard() {
439447
if (read == HttpConstants.LF) {
440448
currentStatus = MultiPartStatus.PREEPILOGUE;
441449
ampersandpos = currentpos - 2;
442-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
450+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
443451
firstpos = currentpos;
444452
contRead = false;
445453
} else {
@@ -452,7 +460,7 @@ private void parseBodyAttributesStandard() {
452460
} else if (read == HttpConstants.LF) {
453461
currentStatus = MultiPartStatus.PREEPILOGUE;
454462
ampersandpos = currentpos - 1;
455-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
463+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
456464
firstpos = currentpos;
457465
contRead = false;
458466
}
@@ -466,15 +474,15 @@ private void parseBodyAttributesStandard() {
466474
// special case
467475
ampersandpos = currentpos;
468476
if (ampersandpos > firstpos) {
469-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
477+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
470478
} else if (!currentAttribute.isCompleted()) {
471479
setFinalBuffer(EMPTY_BUFFER);
472480
}
473481
firstpos = currentpos;
474482
currentStatus = MultiPartStatus.EPILOGUE;
475483
} else if (contRead && currentAttribute != null && currentStatus == MultiPartStatus.FIELD) {
476484
// reset index except if to continue in case of FIELD getStatus
477-
currentAttribute.addContent(undecodedChunk.copy(firstpos, currentpos - firstpos),
485+
currentAttribute.addContent(undecodedChunk.retainedSlice(firstpos, currentpos - firstpos),
478486
false);
479487
firstpos = currentpos;
480488
}
@@ -546,7 +554,7 @@ private void parseBodyAttributes() {
546554
if (read == '&') {
547555
currentStatus = MultiPartStatus.DISPOSITION;
548556
ampersandpos = currentpos - 1;
549-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
557+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
550558
firstpos = currentpos;
551559
contRead = true;
552560
} else if (read == HttpConstants.CR) {
@@ -557,7 +565,7 @@ private void parseBodyAttributes() {
557565
currentStatus = MultiPartStatus.PREEPILOGUE;
558566
ampersandpos = currentpos - 2;
559567
sao.setReadPosition(0);
560-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
568+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
561569
firstpos = currentpos;
562570
contRead = false;
563571
break loop;
@@ -575,7 +583,7 @@ private void parseBodyAttributes() {
575583
currentStatus = MultiPartStatus.PREEPILOGUE;
576584
ampersandpos = currentpos - 1;
577585
sao.setReadPosition(0);
578-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
586+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
579587
firstpos = currentpos;
580588
contRead = false;
581589
break loop;
@@ -592,15 +600,15 @@ private void parseBodyAttributes() {
592600
// special case
593601
ampersandpos = currentpos;
594602
if (ampersandpos > firstpos) {
595-
setFinalBuffer(undecodedChunk.copy(firstpos, ampersandpos - firstpos));
603+
setFinalBuffer(undecodedChunk.retainedSlice(firstpos, ampersandpos - firstpos));
596604
} else if (!currentAttribute.isCompleted()) {
597605
setFinalBuffer(EMPTY_BUFFER);
598606
}
599607
firstpos = currentpos;
600608
currentStatus = MultiPartStatus.EPILOGUE;
601609
} else if (contRead && currentAttribute != null && currentStatus == MultiPartStatus.FIELD) {
602610
// reset index except if to continue in case of FIELD getStatus
603-
currentAttribute.addContent(undecodedChunk.copy(firstpos, currentpos - firstpos),
611+
currentAttribute.addContent(undecodedChunk.retainedSlice(firstpos, currentpos - firstpos),
604612
false);
605613
firstpos = currentpos;
606614
}
@@ -622,8 +630,10 @@ private void parseBodyAttributes() {
622630

623631
private void setFinalBuffer(ByteBuf buffer) throws IOException {
624632
currentAttribute.addContent(buffer, true);
625-
String value = decodeAttribute(currentAttribute.getByteBuf().toString(charset), charset);
626-
currentAttribute.setValue(value);
633+
ByteBuf decodedBuf = decodeAttribute(currentAttribute.getByteBuf(), charset);
634+
if (decodedBuf != null) { // override content only when ByteBuf needed decoding
635+
currentAttribute.setContent(decodedBuf);
636+
}
627637
addHttpData(currentAttribute);
628638
currentAttribute = null;
629639
}
@@ -641,6 +651,28 @@ private static String decodeAttribute(String s, Charset charset) {
641651
}
642652
}
643653

654+
private static ByteBuf decodeAttribute(ByteBuf b, Charset charset) {
655+
int firstEscaped = b.forEachByte(new ByteProcessor.IndexOfProcessor((byte) '%'));
656+
if (firstEscaped == -1) {
657+
return null; // nothing to decode
658+
}
659+
660+
ByteBuf buf = b.alloc().buffer(b.readableBytes());
661+
UrlDecoder urlDecode = new UrlDecoder(buf);
662+
int idx = b.forEachByte(urlDecode);
663+
if (urlDecode.nextEscapedIdx != 0) { // incomplete hex byte
664+
if (idx == -1) {
665+
idx = b.readableBytes() - 1;
666+
}
667+
idx -= urlDecode.nextEscapedIdx - 1;
668+
buf.release();
669+
throw new ErrorDataDecoderException(
670+
String.format("Invalid hex byte at index '%d' in string: '%s'", idx, b.toString(charset)));
671+
}
672+
673+
return buf;
674+
}
675+
644676
/**
645677
* Destroy the {@link HttpPostStandardRequestDecoder} and release all it resources. After this method
646678
* was called it is not possible to operate on it anymore.
@@ -677,4 +709,39 @@ public void removeHttpDataFromClean(InterfaceHttpData data) {
677709

678710
factory.removeHttpDataFromClean(request, data);
679711
}
712+
713+
private static final class UrlDecoder implements ByteProcessor {
714+
715+
private final ByteBuf output;
716+
private int nextEscapedIdx;
717+
private byte hiByte;
718+
719+
UrlDecoder(ByteBuf output) {
720+
this.output = output;
721+
}
722+
723+
@Override
724+
public boolean process(byte value) {
725+
if (nextEscapedIdx != 0) {
726+
if (nextEscapedIdx == 1) {
727+
hiByte = value;
728+
++nextEscapedIdx;
729+
} else {
730+
int hi = StringUtil.decodeHexNibble((char) hiByte);
731+
int lo = StringUtil.decodeHexNibble((char) value);
732+
if (hi == -1 || lo == -1) {
733+
++nextEscapedIdx;
734+
return false;
735+
}
736+
output.writeByte((hi << 4) + lo);
737+
nextEscapedIdx = 0;
738+
}
739+
} else if (value == '%') {
740+
nextEscapedIdx = 1;
741+
} else {
742+
output.writeByte(value);
743+
}
744+
return true;
745+
}
746+
}
680747
}

0 commit comments

Comments
 (0)