Skip to content

Commit e3289c1

Browse files
authored
[feature](lance) support lance catalog in doris (#65730)
### What problem does this PR solve? Issue #66340 Problem Summary: doc: apache/doris-website#4024 This PR introduces read support for the Lance format in Doris, including: 1. Querying Lance datasets through a Lance Catalog. 2. Pushing supported predicates down to Lance to reduce the amount of data scanned. 3. Reading Lance datasets directly through S3 and Local TVFs. 4. Performing vector similarity search through the vector_search() TVF, with the distance exposed through the _distance column. 5. Lance is currently read-only and uses File Scanner V2. Writing to Lance is not supported yet. ``` -- Create a Lance Catalog CREATE CATALOG lance_catalog PROPERTIES ( "type" = "lance", "lance.catalog.type" = "filesystem", "warehouse" = "s3://warehouse/lance", "s3.endpoint" = "http://127.0.0.1:9000", "s3.access_key" = "admin", "s3.secret_key" = "password", "s3.region" = "us-east-1", "use_path_style" = "true" ); -- Regular query; supported WHERE predicates are pushed down to Lance SELECT user_id, user_name FROM lance_catalog.doris.user_profiles WHERE engagement_score >= 90; -- Read a Lance dataset directly through the S3 TVF SELECT user_id, user_name FROM s3( "uri" = "s3://warehouse/lance/doris/user_profiles.lance", "format" = "lance", "s3.endpoint" = "http://127.0.0.1:9000", "s3.access_key" = "admin", "s3.secret_key" = "password", "s3.region" = "us-east-1", "use_path_style" = "true" ); -- Perform a vector similarity search SELECT row_id, label, _distance FROM vector_search( "table" = "lance_catalog.doris.vector_search", "column" = "embedding", "query_vector" = "[0,0,0,0]", "top_k" = "5", "metric" = "l2" ) ORDER BY _distance, row_id; ``` None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [x] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [x] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [x] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent cc2d6a8 commit e3289c1

130 files changed

Lines changed: 12324 additions & 122 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/core/data_type_serde/arrow_validation.h‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,15 @@ inline std::shared_ptr<arrow::Int32Array> get_int32_offsets_array(const arrow::A
9393
return offsets;
9494
}
9595

96+
inline std::shared_ptr<arrow::Int64Array> get_int64_offsets_array(const arrow::Array& array) {
97+
auto offsets_array = static_cast<const arrow::LargeListArray&>(array).offsets();
98+
auto offsets = std::dynamic_pointer_cast<arrow::Int64Array>(offsets_array);
99+
if (UNLIKELY(!offsets)) {
100+
throw_invalid_arrow(array, "offsets array is not Int64Array");
101+
}
102+
return offsets;
103+
}
104+
96105
} // namespace arrow_validation_detail
97106

98107
inline void check_arrow_no_offset(const arrow::Array& array) {
@@ -210,31 +219,33 @@ inline void check_arrow_value_range(const arrow::Array& array, int64_t offset, i
210219
namespace arrow_validation_detail {
211220

212221
// Offsets buffers may come from external Arrow producers through Buffer::Wrap or FFI and are not
213-
// guaranteed to be aligned to int32_t. Do not use Int32Array::Value() here because it performs a
222+
// guaranteed to be naturally aligned. Do not use NumericArray::Value() here because it performs a
214223
// typed raw_values()[i] load and can trigger UBSan on misaligned buffers. Keep this validation path
215224
// consistent with the array/map readers below, which load offsets through unaligned_load().
216-
inline int32_t read_int32_offset(const arrow::Int32Array& offsets, int64_t index) {
225+
template <typename OffsetType, typename ArrowOffsetArray>
226+
inline OffsetType read_offset(const ArrowOffsetArray& offsets, int64_t index) {
217227
const auto* data = reinterpret_cast<const uint8_t*>(offsets.raw_values());
218-
return unaligned_load<int32_t>(data + index * sizeof(int32_t));
228+
return unaligned_load<OffsetType>(data + index * sizeof(OffsetType));
219229
}
220230

221-
inline int64_t check_arrow_offsets_range(const arrow::Int32Array& offsets, int64_t start,
231+
template <typename OffsetType, typename ArrowOffsetArray>
232+
inline int64_t check_arrow_offsets_range(const ArrowOffsetArray& offsets, int64_t start,
222233
int64_t end) {
223234
check_arrow_array_range(offsets, 0, offsets.length());
224-
check_arrow_fixed_width_buffer(offsets, sizeof(int32_t));
235+
check_arrow_fixed_width_buffer(offsets, sizeof(OffsetType));
225236
if (UNLIKELY(start < 0 || end < start || end >= offsets.length())) {
226237
arrow_validation_detail::throw_invalid_arrow(
227238
offsets, "offsets read range is invalid: start={}, end={}, offsets_length={}",
228239
start, end, offsets.length());
229240
}
230241

231-
int64_t previous_offset = read_int32_offset(offsets, start);
242+
int64_t previous_offset = read_offset<OffsetType>(offsets, start);
232243
if (UNLIKELY(previous_offset < 0)) {
233244
arrow_validation_detail::throw_invalid_arrow(
234245
offsets, "offsets contain negative value: offset[{}]={}", start, previous_offset);
235246
}
236247
for (int64_t i = start + 1; i <= end; ++i) {
237-
const int64_t current_offset = read_int32_offset(offsets, i);
248+
const int64_t current_offset = read_offset<OffsetType>(offsets, i);
238249
if (UNLIKELY(current_offset < previous_offset)) {
239250
arrow_validation_detail::throw_invalid_arrow(
240251
offsets,
@@ -253,7 +264,22 @@ inline void check_arrow_list_offsets(const arrow::ListArray& array, int64_t star
253264
check_arrow_array_range(array, start, end);
254265
const auto offsets = arrow_validation_detail::get_int32_offsets_array(array);
255266
const int64_t last_offset =
256-
arrow_validation_detail::check_arrow_offsets_range(*offsets, start, end);
267+
arrow_validation_detail::check_arrow_offsets_range<int32_t>(*offsets, start, end);
268+
const int64_t values_length = array.values() ? array.values()->length() : 0;
269+
if (UNLIKELY(last_offset > values_length)) {
270+
arrow_validation_detail::throw_invalid_arrow(
271+
array, "offsets exceed values length: last_offset={}, values_length={}",
272+
last_offset, values_length);
273+
}
274+
}
275+
276+
// Validate LargeList offsets before reading offsets or recursing into values.
277+
inline void check_arrow_large_list_offsets(const arrow::LargeListArray& array, int64_t start,
278+
int64_t end) {
279+
check_arrow_array_range(array, start, end);
280+
const auto offsets = arrow_validation_detail::get_int64_offsets_array(array);
281+
const int64_t last_offset =
282+
arrow_validation_detail::check_arrow_offsets_range<int64_t>(*offsets, start, end);
257283
const int64_t values_length = array.values() ? array.values()->length() : 0;
258284
if (UNLIKELY(last_offset > values_length)) {
259285
arrow_validation_detail::throw_invalid_arrow(
@@ -267,7 +293,7 @@ inline void check_arrow_map_offsets(const arrow::MapArray& array, int64_t start,
267293
check_arrow_array_range(array, start, end);
268294
const auto offsets = arrow_validation_detail::get_int32_offsets_array(array);
269295
const int64_t last_offset =
270-
arrow_validation_detail::check_arrow_offsets_range(*offsets, start, end);
296+
arrow_validation_detail::check_arrow_offsets_range<int32_t>(*offsets, start, end);
271297
const int64_t keys_length = array.keys() ? array.keys()->length() : 0;
272298
if (UNLIKELY(last_offset > keys_length)) {
273299
arrow_validation_detail::throw_invalid_arrow(

‎be/src/core/data_type_serde/data_type_array_serde.cpp‎

Lines changed: 62 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -326,32 +326,68 @@ Status DataTypeArraySerDe::read_column_from_arrow(IColumn& column, const arrow::
326326
const cctz::time_zone& ctz) const {
327327
auto& column_array = static_cast<ColumnArray&>(column);
328328
auto& offsets_data = column_array.get_offsets();
329-
const auto* concrete_array = dynamic_cast<const arrow::ListArray*>(arrow_array);
330-
auto arrow_offsets_array = concrete_array->offsets();
331-
auto* arrow_offsets = dynamic_cast<arrow::Int32Array*>(arrow_offsets_array.get());
332-
if (config::enable_arrow_input_validation) {
333-
check_arrow_list_offsets(*concrete_array, start, end);
334-
}
335-
auto prev_size = offsets_data.back();
336-
const auto* base_offsets_ptr = reinterpret_cast<const uint8_t*>(arrow_offsets->raw_values());
337-
const size_t offset_element_size = sizeof(int32_t);
338-
int32_t arrow_nested_start_offset = 0;
339-
int32_t arrow_nested_end_offset = 0;
340-
const uint8_t* start_offset_ptr = base_offsets_ptr + start * offset_element_size;
341-
const uint8_t* end_offset_ptr = base_offsets_ptr + end * offset_element_size;
342-
memcpy(&arrow_nested_start_offset, start_offset_ptr, offset_element_size);
343-
memcpy(&arrow_nested_end_offset, end_offset_ptr, offset_element_size);
344-
345-
for (auto i = start + 1; i < end + 1; ++i) {
346-
int32_t current_offset = 0;
347-
const uint8_t* current_offset_ptr = base_offsets_ptr + i * offset_element_size;
348-
memcpy(&current_offset, current_offset_ptr, offset_element_size);
349-
// convert to doris offset, start from offsets.back()
350-
offsets_data.emplace_back(prev_size + current_offset - arrow_nested_start_offset);
351-
}
352-
return nested_serde->read_column_from_arrow(
353-
column_array.get_data(), concrete_array->values().get(), arrow_nested_start_offset,
354-
arrow_nested_end_offset, ctz);
329+
330+
const auto read_list = [&](const auto* concrete_array, const auto& read_offset) -> Status {
331+
const int64_t arrow_nested_start_offset = read_offset(start);
332+
const int64_t arrow_nested_end_offset = read_offset(end);
333+
const auto prev_size = offsets_data.back();
334+
for (int64_t i = start + 1; i <= end; ++i) {
335+
// Convert Arrow offsets to Doris offsets, starting at offsets.back().
336+
offsets_data.emplace_back(prev_size + read_offset(i) - arrow_nested_start_offset);
337+
}
338+
return nested_serde->read_column_from_arrow(
339+
column_array.get_data(), concrete_array->values().get(), arrow_nested_start_offset,
340+
arrow_nested_end_offset, ctz);
341+
};
342+
343+
switch (arrow_array->type_id()) {
344+
case arrow::Type::LIST: {
345+
const auto* concrete_array = dynamic_cast<const arrow::ListArray*>(arrow_array);
346+
if (concrete_array == nullptr) {
347+
return Status::InvalidArgument("Expected Arrow ListArray, got {}",
348+
arrow_array->type()->name());
349+
}
350+
if (config::enable_arrow_input_validation) {
351+
check_arrow_list_offsets(*concrete_array, start, end);
352+
}
353+
const auto* offsets = concrete_array->value_offsets()->data();
354+
const auto array_offset = concrete_array->offset();
355+
return read_list(concrete_array, [offsets, array_offset](int64_t index) {
356+
return unaligned_load<int32_t>(offsets + (array_offset + index) * sizeof(int32_t));
357+
});
358+
}
359+
case arrow::Type::LARGE_LIST: {
360+
const auto* concrete_array = dynamic_cast<const arrow::LargeListArray*>(arrow_array);
361+
if (concrete_array == nullptr) {
362+
return Status::InvalidArgument("Expected Arrow LargeListArray, got {}",
363+
arrow_array->type()->name());
364+
}
365+
if (config::enable_arrow_input_validation) {
366+
check_arrow_large_list_offsets(*concrete_array, start, end);
367+
}
368+
const auto* offsets = concrete_array->value_offsets()->data();
369+
const auto array_offset = concrete_array->offset();
370+
return read_list(concrete_array, [offsets, array_offset](int64_t index) {
371+
return unaligned_load<int64_t>(offsets + (array_offset + index) * sizeof(int64_t));
372+
});
373+
}
374+
case arrow::Type::FIXED_SIZE_LIST: {
375+
const auto* concrete_array = dynamic_cast<const arrow::FixedSizeListArray*>(arrow_array);
376+
if (concrete_array == nullptr) {
377+
return Status::InvalidArgument("Expected Arrow FixedSizeListArray, got {}",
378+
arrow_array->type()->name());
379+
}
380+
if (config::enable_arrow_input_validation) {
381+
check_arrow_array_range(*concrete_array, start, end);
382+
}
383+
return read_list(concrete_array, [concrete_array](int64_t index) {
384+
return concrete_array->value_offset(index);
385+
});
386+
}
387+
default:
388+
return Status::InvalidArgument("Unsupported Arrow array type for Doris ARRAY: {}",
389+
arrow_array->type()->name());
390+
}
355391
}
356392

357393
Status DataTypeArraySerDe::write_column_to_mysql_binary(const IColumn& column,

‎be/src/core/data_type_serde/data_type_datetimev2_serde.cpp‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -619,22 +619,32 @@ Status DataTypeDateTimeV2SerDe::read_column_from_arrow(IColumn& column,
619619
type->unit());
620620
}
621621
}
622+
// A timezone-naive Arrow timestamp is a wall-clock value. Decode its epoch-based
623+
// representation in UTC so the session timezone does not shift its date/time fields.
624+
const cctz::time_zone& real_ctz = type->timezone().empty() ? cctz::utc_time_zone() : ctz;
622625
const auto* base_ptr = reinterpret_cast<const uint8_t*>(concrete_array->raw_values());
623626
const size_t element_size = sizeof(int64_t);
624627
for (auto value_i = start; value_i < end; ++value_i) {
625628
int64_t date_value = 0;
626629
const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size;
627630
memcpy(&date_value, raw_byte_ptr, element_size);
628-
auto utc_epoch = static_cast<UInt64>(date_value);
629-
630631
DateV2Value<DateTimeV2ValueType> v;
631-
// convert second
632-
v.from_unixtime(utc_epoch / divisor, ctz);
633-
// get rest time
632+
// C++ integer division truncates toward zero. Normalize the remainder so a negative
633+
// timestamp still has a non-negative fractional part, e.g. -876544us becomes
634+
// -1 second and 123456us.
635+
int64_t seconds = date_value / divisor;
636+
int64_t remainder = date_value % divisor;
637+
if (remainder < 0) {
638+
--seconds;
639+
remainder += divisor;
640+
}
641+
v.from_unixtime(seconds, real_ctz);
642+
// Get the fractional part.
634643
// add 0 on the right to make it 6 digits. DateTimeV2Value microsecond is 6 digits,
635644
// the scale decides to keep the first few digits, so the valid digits should be kept at the front.
636-
// "2022-01-01 11:11:11.111", utc_epoch = 1641035471111, divisor = 1000, set_microsecond(111000)
637-
v.set_microsecond((utc_epoch % divisor) * DIVISOR_FOR_MICRO / divisor);
645+
// "2022-01-01 11:11:11.111", timestamp = 1641035471111, divisor = 1000,
646+
// set_microsecond(111000)
647+
v.set_microsecond(remainder * DIVISOR_FOR_MICRO / divisor);
638648
col_data.emplace_back(v);
639649
}
640650
} else {

‎be/src/core/data_type_serde/data_type_datev2_serde.cpp‎

Lines changed: 53 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -204,17 +204,59 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
204204
check_arrow_no_offset(*arrow_array);
205205
}
206206
auto& col_data = static_cast<ColumnDateV2&>(column).get_data();
207-
const auto* concrete_array = dynamic_cast<const arrow::Date32Array*>(arrow_array);
208-
const auto* base_ptr = reinterpret_cast<const uint8_t*>(concrete_array->raw_values());
209-
const size_t element_size = sizeof(int32_t);
210-
for (auto value_i = start; value_i < end; ++value_i) {
211-
int32_t date_value = 0;
212-
const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size;
213-
memcpy(&date_value, raw_byte_ptr, element_size);
214-
215-
DateV2Value<DateV2ValueType> v;
216-
v.get_date_from_daynr(date_value + date_threshold);
217-
col_data.emplace_back(v);
207+
if (arrow_array->type_id() == arrow::Type::DATE64) {
208+
static constexpr int64_t MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
209+
const auto* concrete_array = dynamic_cast<const arrow::Date64Array*>(arrow_array);
210+
if (concrete_array == nullptr) {
211+
return Status::InvalidArgument("Expected Arrow Date64Array, got {}",
212+
arrow_array->type()->name());
213+
}
214+
if (config::enable_arrow_input_validation) {
215+
check_arrow_array_range(*concrete_array, start, end);
216+
check_arrow_fixed_width_buffer(*concrete_array, sizeof(arrow::Date64Array::value_type));
217+
}
218+
for (int64_t value_i = start; value_i < end; ++value_i) {
219+
if (concrete_array->IsNull(value_i)) {
220+
col_data.emplace_back(DateV2Value<DateV2ValueType>());
221+
continue;
222+
}
223+
const int64_t milliseconds = concrete_array->Value(value_i);
224+
if (milliseconds % MILLISECONDS_PER_DAY != 0) {
225+
return Status::InvalidArgument(
226+
"Arrow Date64 value must contain whole days: row={}, milliseconds={}",
227+
value_i, milliseconds);
228+
}
229+
const int64_t daynr = milliseconds / MILLISECONDS_PER_DAY + date_threshold;
230+
DateV2Value<DateV2ValueType> value;
231+
if (daynr <= 0 || daynr > DATE_MAX_DAYNR ||
232+
!value.get_date_from_daynr(static_cast<uint64_t>(daynr))) {
233+
return Status::InvalidArgument(
234+
"Arrow Date64 value is outside the Doris DATE range: "
235+
"row={}, milliseconds={}",
236+
value_i, milliseconds);
237+
}
238+
col_data.emplace_back(value);
239+
}
240+
} else if (arrow_array->type_id() == arrow::Type::DATE32) {
241+
const auto* concrete_array = dynamic_cast<const arrow::Date32Array*>(arrow_array);
242+
if (concrete_array == nullptr) {
243+
return Status::InvalidArgument("Expected Arrow Date32Array, got {}",
244+
arrow_array->type()->name());
245+
}
246+
const auto* base_ptr = reinterpret_cast<const uint8_t*>(concrete_array->raw_values());
247+
const size_t element_size = sizeof(int32_t);
248+
for (auto value_i = start; value_i < end; ++value_i) {
249+
int32_t date_value = 0;
250+
const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size;
251+
memcpy(&date_value, raw_byte_ptr, element_size);
252+
253+
DateV2Value<DateV2ValueType> v;
254+
v.get_date_from_daynr(date_value + date_threshold);
255+
col_data.emplace_back(v);
256+
}
257+
} else {
258+
return Status::InvalidArgument("Expected Arrow Date32Array or Date64Array, got {}",
259+
arrow_array->type()->name());
218260
}
219261
return Status::OK();
220262
}

0 commit comments

Comments
 (0)