Skip to content

Commit 1508918

Browse files
sollhuipull[bot]
authored andcommitted
[feature](streamload) support HTTP request with chunked transfer (#26520)
1 parent b782d03 commit 1508918

6 files changed

Lines changed: 56 additions & 4 deletions

File tree

‎be/src/http/action/stream_load.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(streaming_load_requests_total, MetricUnit::
7676
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(streaming_load_duration_ms, MetricUnit::MILLISECONDS);
7777
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(streaming_load_current_processing, MetricUnit::REQUESTS);
7878

79+
static constexpr size_t MIN_CHUNK_SIZE = 64 * 1024;
80+
static const string CHUNK = "chunked";
81+
7982
#ifdef BE_TEST
8083
TStreamLoadPutResult k_stream_load_put_result;
8184
#endif
@@ -292,6 +295,12 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptr<Strea
292295
#endif
293296
}
294297

298+
if (!http_req->header(HttpHeaders::TRANSFER_ENCODING).empty()) {
299+
if (http_req->header(HttpHeaders::TRANSFER_ENCODING).find(CHUNK) != std::string::npos) {
300+
ctx->is_chunked_transfer = true;
301+
}
302+
}
303+
295304
if (!http_req->header(HTTP_TIMEOUT).empty()) {
296305
try {
297306
ctx->timeout_second = std::stoi(http_req->header(HTTP_TIMEOUT));
@@ -369,9 +378,15 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
369378
request.__set_header_type(ctx->header_type);
370379
request.__set_loadId(ctx->id.to_thrift());
371380
if (ctx->use_streaming) {
372-
auto pipe = std::make_shared<io::StreamLoadPipe>(
373-
io::kMaxPipeBufferedBytes /* max_buffered_bytes */, 64 * 1024 /* min_chunk_size */,
374-
ctx->body_bytes /* total_length */);
381+
std::shared_ptr<io::StreamLoadPipe> pipe;
382+
if (ctx->is_chunked_transfer) {
383+
pipe = std::make_shared<io::StreamLoadPipe>(
384+
io::kMaxPipeBufferedBytes /* max_buffered_bytes */);
385+
} else {
386+
pipe = std::make_shared<io::StreamLoadPipe>(
387+
io::kMaxPipeBufferedBytes /* max_buffered_bytes */,
388+
MIN_CHUNK_SIZE /* min_chunk_size */, ctx->body_bytes /* total_length */);
389+
}
375390
request.fileType = TFileType::FILE_STREAM;
376391
ctx->body_sink = pipe;
377392
ctx->pipe = pipe;

‎be/src/io/fs/stream_load_pipe.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ Status StreamLoadPipe::read_at_impl(size_t /*offset*/, Slice result, size_t* byt
9090
return Status::OK();
9191
}
9292

93-
// If _total_length == -1, this should be a Kafka routine load task,
93+
// If _total_length == -1, this should be a Kafka routine load task or stream load with chunked transfer HTTP request,
9494
// just get the next buffer directly from the buffer queue, because one buffer contains a complete piece of data.
9595
// Otherwise, this should be a stream load task that needs to read the specified amount of data.
9696
Status StreamLoadPipe::read_one_message(std::unique_ptr<uint8_t[]>* data, size_t* length) {

‎be/src/runtime/stream_load/stream_load_context.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ class StreamLoadContext {
161161
// only used to check if we receive whole body
162162
size_t body_bytes = 0;
163163
size_t receive_bytes = 0;
164+
bool is_chunked_transfer = false;
164165

165166
int64_t txn_id = default_txn_id;
166167

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
2|-50|1|44|1
2+
1|-50|1|2|1

‎regression-test/data/load_p0/stream_load/test_stream_load.out‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,3 +113,7 @@
113113
1 -50 1 2 1 \N
114114
2 -50 1 44 1 \N
115115

116+
-- !sql_chunked_transfer --
117+
1 -50 1 2 1 \N
118+
2 -50 1 44 1 \N
119+

‎regression-test/suites/load_p0/stream_load/test_stream_load.groovy‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1134,5 +1134,35 @@ suite("test_stream_load", "p0") {
11341134
} finally {
11351135
sql """ DROP TABLE IF EXISTS ${tableName15} FORCE"""
11361136
}
1137+
1138+
// test chunked transfer
1139+
def tableName16 = "test_chunked_transfer"
1140+
try {
1141+
sql """ DROP TABLE IF EXISTS ${tableName16} """
1142+
sql """
1143+
CREATE TABLE IF NOT EXISTS ${tableName16} (
1144+
`k1` bigint(20) NULL DEFAULT "1",
1145+
`k2` bigint(20) NULL ,
1146+
`v1` tinyint(4) NULL,
1147+
`v2` tinyint(4) NULL,
1148+
`v3` tinyint(4) NULL,
1149+
`v4` DATETIME NULL
1150+
) ENGINE=OLAP
1151+
DISTRIBUTED BY HASH(`k1`) BUCKETS 3
1152+
PROPERTIES ("replication_allocation" = "tag.location.default: 1");
1153+
"""
1154+
1155+
def command = "curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword} -H column_separator:| -H Transfer-Encoding:chunked -H columns:k1,k2,v1,v2,v3 -T ${context.dataPath}/test_chunked_transfer.csv http://${context.config.feHttpAddress}/api/${db}/${tableName16}/_stream_load"
1156+
log.info("test chunked transfer command: ${command}")
1157+
def process = command.execute()
1158+
code = process.waitFor()
1159+
out = process.text
1160+
json2pc = parseJson(out)
1161+
log.info("test chunked transfer result: ${out}".toString())
1162+
1163+
qt_sql_chunked_transfer "select * from ${tableName16} order by k1"
1164+
} finally {
1165+
sql """ DROP TABLE IF EXISTS ${tableName16} FORCE"""
1166+
}
11371167
}
11381168

0 commit comments

Comments
 (0)