|
47 | 47 | #include "core/value/uuid_value.h" |
48 | 48 | #include "core/value/variant/variant_batch_builder.h" |
49 | 49 | #include "core/value/variant/variant_parquet_encoding.h" |
| 50 | +#include "exec/common/variant_util.h" |
50 | 51 | #include "exprs/function/search/function_search.h" |
51 | 52 | #include "exprs/function/search/variant_inverted_index_search.h" |
52 | 53 | #include "exprs/vectorized_fn_call.h" |
|
79 | 80 | #include "storage/segment/variant/variant_column_reader.h" |
80 | 81 | #include "storage/segment/variant/variant_column_writer_impl.h" |
81 | 82 | #include "storage/segment/variant/variant_doc_snpashot_compact_iterator.h" |
| 83 | +#include "storage/segment/variant/variant_writer_helpers.h" |
82 | 84 | #include "storage/storage_engine.h" |
83 | 85 | #include "storage/tablet/tablet_manager.h" |
84 | 86 | #include "testutil/variant_util.h" |
@@ -1872,9 +1874,13 @@ class VariantColumnWriterReaderTest : public testing::Test { |
1872 | 1874 | RETURN_IF_ERROR(ColumnReader::create(reader_options, footer.columns(0), footer.num_rows(), |
1873 | 1875 | file_reader, &reader)); |
1874 | 1876 |
|
1875 | | - TabletColumn read_column = _tablet_schema->column(1); |
1876 | | - read_column.set_type(static_cast<FieldType>(footer.columns(0).type())); |
1877 | | - auto read_type = DataTypeFactory::instance().create_data_type(read_column, false); |
| 1877 | + // Build the read type and the matching tablet column from the footer meta so nested types |
| 1878 | + // (ARRAY/MAP/STRUCT) keep their child type information; the schema column only carries the |
| 1879 | + // top-level type and the array iterator needs the element sub-column. |
| 1880 | + auto read_type = DataTypeFactory::instance().create_data_type(footer.columns(0)); |
| 1881 | + TabletColumn read_column = variant_util::get_column_by_type( |
| 1882 | + read_type, _tablet_schema->column(1).name(), |
| 1883 | + variant_util::ExtraInfo {.unique_id = -1, .parent_unique_id = 1}); |
1878 | 1884 | ColumnIteratorUPtr iterator; |
1879 | 1885 | RETURN_IF_ERROR(reader->new_iterator(&iterator, &read_column)); |
1880 | 1886 | OlapReaderStatistics stats; |
@@ -2489,8 +2495,9 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_root_writer_streams_per_batch |
2489 | 2495 | converter.clear_source_content(0); |
2490 | 2496 | max_buffered_bytes = std::max<uint64_t>(max_buffered_bytes, writer->estimate_buffer_size()); |
2491 | 2497 | } |
2492 | | - // The buffering writer retained the whole segment (kRows * kValueBytes ~ 16 MiB); the |
2493 | | - // streaming writer holds at most the current page plus the outer-null map. |
| 2498 | + // The buffering writer retained the whole segment (kRows * kValueBytes ~ 16 MiB). The |
| 2499 | + // streaming writer flushes the root column batch by batch, so its own retained raw buffer is |
| 2500 | + // about one batch; encoded pages and the other variant column writers are separate. |
2494 | 2501 | EXPECT_LT(max_buffered_bytes, 2 * kBatchRows * kValueBytes); |
2495 | 2502 |
|
2496 | 2503 | ASSERT_TRUE(writer->finish().ok()); |
@@ -9252,4 +9259,100 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_by_by |
9252 | 9259 | EXPECT_EQ(chunked_rows, ref_rows); |
9253 | 9260 | } |
9254 | 9261 |
|
| 9262 | +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_int_then_bool) { |
| 9263 | + // The single builder falls back to JSONB when a bool follows integers: appending `true` into |
| 9264 | + // an Int64 column is not a representable promotion. Chunks must carry the inferred type across |
| 9265 | + // their boundary so the same value sequence keeps the JSONB fallback instead of coercing |
| 9266 | + // true/false to 1/0 through the least-common-type merge. |
| 9267 | + init_extracted_variant_schema(11406); |
| 9268 | + const std::vector<std::string> jsons {"7", "true", "false", "8"}; |
| 9269 | + ColumnPtr source; |
| 9270 | + DataTypePtr source_type; |
| 9271 | + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source, |
| 9272 | + &source_type) |
| 9273 | + .ok()); |
| 9274 | + |
| 9275 | + SegmentFooterPB ref_footer; |
| 9276 | + std::string ref_path; |
| 9277 | + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_bool_ref", |
| 9278 | + &ref_footer, &ref_path) |
| 9279 | + .ok()); |
| 9280 | + std::vector<std::string> ref_rows; |
| 9281 | + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); |
| 9282 | + |
| 9283 | + struct ChunkGuard { |
| 9284 | + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(1); } |
| 9285 | + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } |
| 9286 | + } guard; |
| 9287 | + size_t max_chunks = 0; |
| 9288 | + SegmentFooterPB chunked_footer; |
| 9289 | + std::string chunked_path; |
| 9290 | + ASSERT_TRUE(write_extracted_variant_segment( |
| 9291 | + source, source_type, "subcolumn_bool_chunked", &chunked_footer, |
| 9292 | + &chunked_path, 0, 1, |
| 9293 | + [&](ColumnWriter* writer) { |
| 9294 | + auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer); |
| 9295 | + ASSERT_NE(subcolumn, nullptr); |
| 9296 | + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); |
| 9297 | + }) |
| 9298 | + .ok()); |
| 9299 | + EXPECT_GE(max_chunks, 2); |
| 9300 | + |
| 9301 | + std::vector<std::string> chunked_rows; |
| 9302 | + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); |
| 9303 | + EXPECT_EQ(chunked_rows, ref_rows); |
| 9304 | + EXPECT_EQ(chunked_rows, (std::vector<std::string> {"7", "true", "false", "8"})); |
| 9305 | + EXPECT_EQ(static_cast<FieldType>(ref_footer.columns(0).type()), |
| 9306 | + FieldType::OLAP_FIELD_TYPE_JSONB); |
| 9307 | + EXPECT_EQ(static_cast<FieldType>(chunked_footer.columns(0).type()), |
| 9308 | + FieldType::OLAP_FIELD_TYPE_JSONB); |
| 9309 | +} |
| 9310 | + |
| 9311 | +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_array_all_null_tail) { |
| 9312 | + // An all-NULL tail chunk of an ARRAY subcolumn must be flushed through the array writer (which |
| 9313 | + // updates offsets, nulls and the inverted index together), not through append_nulls. |
| 9314 | + init_extracted_variant_schema(11407); |
| 9315 | + const std::vector<std::string> jsons {"[1,2]", "[\"a\",\"b\"]", "null", "null"}; |
| 9316 | + ColumnPtr source; |
| 9317 | + DataTypePtr source_type; |
| 9318 | + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source, |
| 9319 | + &source_type) |
| 9320 | + .ok()); |
| 9321 | + |
| 9322 | + SegmentFooterPB ref_footer; |
| 9323 | + std::string ref_path; |
| 9324 | + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_array_ref", |
| 9325 | + &ref_footer, &ref_path) |
| 9326 | + .ok()); |
| 9327 | + std::vector<std::string> ref_rows; |
| 9328 | + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); |
| 9329 | + |
| 9330 | + struct ChunkGuard { |
| 9331 | + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(2); } |
| 9332 | + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } |
| 9333 | + } guard; |
| 9334 | + segment_v2::variant_writer_helpers::reset_array_all_null_chunks_for_test(); |
| 9335 | + size_t max_chunks = 0; |
| 9336 | + SegmentFooterPB chunked_footer; |
| 9337 | + std::string chunked_path; |
| 9338 | + ASSERT_TRUE(write_extracted_variant_segment( |
| 9339 | + source, source_type, "subcolumn_array_chunked", &chunked_footer, |
| 9340 | + &chunked_path, 0, 1, |
| 9341 | + [&](ColumnWriter* writer) { |
| 9342 | + auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer); |
| 9343 | + ASSERT_NE(subcolumn, nullptr); |
| 9344 | + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); |
| 9345 | + }) |
| 9346 | + .ok()); |
| 9347 | + EXPECT_GE(max_chunks, 2); |
| 9348 | + EXPECT_EQ(segment_v2::variant_writer_helpers::array_all_null_chunks_for_test(), 1u); |
| 9349 | + |
| 9350 | + std::vector<std::string> chunked_rows; |
| 9351 | + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); |
| 9352 | + EXPECT_EQ(chunked_rows, ref_rows); |
| 9353 | + ASSERT_EQ(chunked_rows.size(), 4); |
| 9354 | + EXPECT_EQ(chunked_rows[2], "NULL"); |
| 9355 | + EXPECT_EQ(chunked_rows[3], "NULL"); |
| 9356 | +} |
| 9357 | + |
9255 | 9358 | } // namespace doris |
0 commit comments