@@ -76,6 +76,9 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(streaming_load_requests_total, MetricUnit::
7676DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG (streaming_load_duration_ms, MetricUnit::MILLISECONDS );
7777DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG (streaming_load_current_processing, MetricUnit::REQUESTS );
7878
79+ static constexpr size_t MIN_CHUNK_SIZE = 64 * 1024 ;
80+ static const string CHUNK = " chunked" ;
81+
7982#ifdef BE_TEST
8083TStreamLoadPutResult k_stream_load_put_result;
8184#endif
@@ -292,6 +295,12 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptr<Strea
292295#endif
293296 }
294297
298+ if (!http_req->header (HttpHeaders::TRANSFER_ENCODING ).empty ()) {
299+ if (http_req->header (HttpHeaders::TRANSFER_ENCODING ).find (CHUNK ) != std::string::npos) {
300+ ctx->is_chunked_transfer = true ;
301+ }
302+ }
303+
295304 if (!http_req->header (HTTP_TIMEOUT ).empty ()) {
296305 try {
297306 ctx->timeout_second = std::stoi (http_req->header (HTTP_TIMEOUT ));
@@ -369,9 +378,15 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
369378 request.__set_header_type (ctx->header_type );
370379 request.__set_loadId (ctx->id .to_thrift ());
371380 if (ctx->use_streaming ) {
372- auto pipe = std::make_shared<io::StreamLoadPipe>(
373- io::kMaxPipeBufferedBytes /* max_buffered_bytes */ , 64 * 1024 /* min_chunk_size */ ,
374- ctx->body_bytes /* total_length */ );
381+ std::shared_ptr<io::StreamLoadPipe> pipe;
382+ if (ctx->is_chunked_transfer ) {
383+ pipe = std::make_shared<io::StreamLoadPipe>(
384+ io::kMaxPipeBufferedBytes /* max_buffered_bytes */ );
385+ } else {
386+ pipe = std::make_shared<io::StreamLoadPipe>(
387+ io::kMaxPipeBufferedBytes /* max_buffered_bytes */ ,
388+ MIN_CHUNK_SIZE /* min_chunk_size */ , ctx->body_bytes /* total_length */ );
389+ }
375390 request.fileType = TFileType::FILE_STREAM ;
376391 ctx->body_sink = pipe;
377392 ctx->pipe = pipe;
0 commit comments