@@ -1327,6 +1327,25 @@ static Status create_variant_root_reader(const SegmentFooterPB& footer,
13271327
13281328class VariantColumnWriterReaderTest : public testing ::Test {
13291329public:
1330+ // Parent VARIANT column with an extracted VARIANT subcolumn `v.payload`, i.e. the layout
1331+ // VariantSubcolumnWriter is responsible for.
1332+ void init_extracted_variant_schema (int64_t tablet_id) {
1333+ TabletSchemaPB schema_pb;
1334+ schema_pb.set_keys_type (KeysType::DUP_KEYS );
1335+ construct_column (schema_pb.add_column (), 1 , " VARIANT" , " v" , 1 );
1336+ _tablet_schema = std::make_shared<TabletSchema>();
1337+ _tablet_schema->init_from_pb (schema_pb);
1338+ const TabletColumn& parent_column = _tablet_schema->column_by_uid (1 );
1339+ TabletColumn extracted_column;
1340+ extracted_column.set_name (parent_column.name_lower_case () + " .payload" );
1341+ extracted_column.set_type (FieldType::OLAP_FIELD_TYPE_VARIANT );
1342+ extracted_column.set_parent_unique_id (parent_column.unique_id ());
1343+ extracted_column.set_path_info (PathInData (parent_column.name_lower_case () + " .payload" ));
1344+ extracted_column.set_is_nullable (true );
1345+ _tablet_schema->append_column (extracted_column);
1346+ init_tablet_from_current_schema (tablet_id);
1347+ }
1348+
13301349 void SetUp () override {
13311350 // absolute dir
13321351 char buffer[MAX_PATH_LEN ];
@@ -1721,9 +1740,10 @@ class VariantColumnWriterReaderTest : public testing::Test {
17211740 return Status::OK ();
17221741 }
17231742
1724- Status write_extracted_variant_segment (const ColumnPtr& source, const DataTypePtr& source_type,
1725- std::string_view rowset_id, SegmentFooterPB* footer,
1726- std::string* file_path, size_t first_batch_rows = 0 ) {
1743+ Status write_extracted_variant_segment (
1744+ const ColumnPtr& source, const DataTypePtr& source_type, std::string_view rowset_id,
1745+ SegmentFooterPB* footer, std::string* file_path, size_t first_batch_rows = 0 ,
1746+ size_t batch_rows = 0 , const std::function<void (ColumnWriter*)>& after_batch = {}) {
17271747 DORIS_CHECK (source.get () != nullptr );
17281748 DORIS_CHECK (source_type != nullptr );
17291749 DORIS_CHECK (footer != nullptr );
@@ -1768,11 +1788,28 @@ class VariantColumnWriterReaderTest : public testing::Test {
17681788 converter.clear_source_content (0 );
17691789 return Status::OK ();
17701790 };
1771- if (first_batch_rows > 0 && first_batch_rows < source->size ()) {
1791+ if (batch_rows > 0 ) {
1792+ for (size_t row_pos = 0 ; row_pos < source->size (); row_pos += batch_rows) {
1793+ const size_t rows = std::min (batch_rows, source->size () - row_pos);
1794+ RETURN_IF_ERROR (append_batch (row_pos, rows));
1795+ if (after_batch) {
1796+ after_batch (writer.get ());
1797+ }
1798+ }
1799+ } else if (first_batch_rows > 0 && first_batch_rows < source->size ()) {
17721800 RETURN_IF_ERROR (append_batch (0 , first_batch_rows));
1801+ if (after_batch) {
1802+ after_batch (writer.get ());
1803+ }
17731804 RETURN_IF_ERROR (append_batch (first_batch_rows, source->size () - first_batch_rows));
1805+ if (after_batch) {
1806+ after_batch (writer.get ());
1807+ }
17741808 } else {
17751809 RETURN_IF_ERROR (append_batch (0 , source->size ()));
1810+ if (after_batch) {
1811+ after_batch (writer.get ());
1812+ }
17761813 }
17771814
17781815 RETURN_IF_ERROR (writer->finish ());
@@ -8973,4 +9010,246 @@ TEST_F(VariantColumnWriterReaderTest, legacy_v1_segment_compaction_preserves_ful
89739010 EXPECT_EQ (actual_rows, expected_rows);
89749011}
89759012
9013+ TEST_F (VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_large_values) {
9014+ // A large VARIANT subcolumn must be flushed in bounded chunks: a single segment-sized
9015+ // ColumnString is what overflows ColumnStr<uint32_t> at 4 GiB. Compare the chunked output with
9016+ // the single-chunk reference and check the observed chunk budget.
9017+ init_extracted_variant_schema (11401 );
9018+ constexpr size_t kRows = 3000 ;
9019+ constexpr size_t kBatchRows = 300 ;
9020+ const std::string value (1024 , ' x' );
9021+ const std::string json = " \" " + value + " \" " ;
9022+ std::vector<std::string> jsons (kRows , json);
9023+ ColumnPtr source;
9024+ DataTypePtr source_type;
9025+ ASSERT_TRUE (create_variant_writer_source (VariantWriterInput::V2 , jsons, 0 , false , {}, &source,
9026+ &source_type)
9027+ .ok ());
9028+
9029+ SegmentFooterPB ref_footer;
9030+ std::string ref_path;
9031+ ASSERT_TRUE (write_extracted_variant_segment (source, source_type, " subcolumn_chunk_ref" ,
9032+ &ref_footer, &ref_path)
9033+ .ok ());
9034+ std::vector<std::string> ref_rows;
9035+ ASSERT_TRUE (read_extracted_variant_rows (ref_footer, ref_path, &ref_rows).ok ());
9036+ ASSERT_EQ (ref_rows.size (), kRows );
9037+
9038+ struct ChunkGuard {
9039+ ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (250 ); }
9040+ ~ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (0 ); }
9041+ } guard;
9042+ size_t max_chunks = 0 ;
9043+ size_t max_chunk_bytes = 0 ;
9044+ SegmentFooterPB chunked_footer;
9045+ std::string chunked_path;
9046+ ASSERT_TRUE (write_extracted_variant_segment (
9047+ source, source_type, " subcolumn_chunk_values" , &chunked_footer,
9048+ &chunked_path, 0 , kBatchRows ,
9049+ [&](ColumnWriter* writer) {
9050+ auto * subcolumn = dynamic_cast <VariantSubcolumnWriter*>(writer);
9051+ ASSERT_NE (subcolumn, nullptr );
9052+ max_chunks = std::max (max_chunks, subcolumn->num_chunks_for_test ());
9053+ max_chunk_bytes = std::max (max_chunk_bytes,
9054+ subcolumn->max_chunk_bytes_for_test ());
9055+ })
9056+ .ok ());
9057+ EXPECT_GE (max_chunks, 2 );
9058+ // A chunk holds at most the configured budget plus the batch that crossed it.
9059+ EXPECT_LE (max_chunk_bytes, 250 * 1024 + kBatchRows * 1024 );
9060+
9061+ std::vector<std::string> chunked_rows;
9062+ ASSERT_TRUE (read_extracted_variant_rows (chunked_footer, chunked_path, &chunked_rows).ok ());
9063+ EXPECT_EQ (chunked_rows, ref_rows);
9064+ EXPECT_EQ (chunked_rows.front (), ref_rows.front ());
9065+ EXPECT_EQ (chunked_rows.back (), ref_rows.back ());
9066+ }
9067+
9068+ TEST_F (VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_type_promotion) {
9069+ // Values that widen the inferred type across a chunk boundary (int -> string -> JSONB) must
9070+ // still produce the same column as the single-chunk path.
9071+ init_extracted_variant_schema (11402 );
9072+ constexpr size_t kHalf = 2000 ;
9073+ std::vector<std::string> jsons;
9074+ jsons.reserve (2 * kHalf );
9075+ for (size_t i = 0 ; i < kHalf ; ++i) {
9076+ jsons.push_back (std::to_string (i));
9077+ }
9078+ for (size_t i = 0 ; i < kHalf ; ++i) {
9079+ jsons.push_back (" \" s" + std::to_string (i) + " \" " );
9080+ }
9081+ ColumnPtr source;
9082+ DataTypePtr source_type;
9083+ ASSERT_TRUE (create_variant_writer_source (VariantWriterInput::V2 , jsons, 0 , false , {}, &source,
9084+ &source_type)
9085+ .ok ());
9086+
9087+ SegmentFooterPB ref_footer;
9088+ std::string ref_path;
9089+ ASSERT_TRUE (write_extracted_variant_segment (source, source_type, " subcolumn_promote_ref" ,
9090+ &ref_footer, &ref_path)
9091+ .ok ());
9092+ std::vector<std::string> ref_rows;
9093+ ASSERT_TRUE (read_extracted_variant_rows (ref_footer, ref_path, &ref_rows).ok ());
9094+
9095+ struct ChunkGuard {
9096+ ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (500 ); }
9097+ ~ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (0 ); }
9098+ } guard;
9099+ size_t max_chunks = 0 ;
9100+ SegmentFooterPB chunked_footer;
9101+ std::string chunked_path;
9102+ ASSERT_TRUE (write_extracted_variant_segment (
9103+ source, source_type, " subcolumn_promote_chunked" , &chunked_footer,
9104+ &chunked_path, 0 , 300 ,
9105+ [&](ColumnWriter* writer) {
9106+ auto * subcolumn = dynamic_cast <VariantSubcolumnWriter*>(writer);
9107+ ASSERT_NE (subcolumn, nullptr );
9108+ max_chunks = std::max (max_chunks, subcolumn->num_chunks_for_test ());
9109+ })
9110+ .ok ());
9111+ EXPECT_GE (max_chunks, 2 );
9112+
9113+ std::vector<std::string> chunked_rows;
9114+ ASSERT_TRUE (read_extracted_variant_rows (chunked_footer, chunked_path, &chunked_rows).ok ());
9115+ EXPECT_EQ (chunked_rows, ref_rows);
9116+ }
9117+
9118+ TEST_F (VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_sparse_rows) {
9119+ // Rows without the subcolumn become nulls. When chunks split a gap the dense materialization
9120+ // must still place every value on its logical row.
9121+ init_extracted_variant_schema (11403 );
9122+ constexpr size_t kRows = 3000 ;
9123+ std::vector<std::string> jsons (kRows , " \" v\" " );
9124+ std::vector<UInt8> outer_nulls (kRows , 0 );
9125+ for (size_t row = 0 ; row < kRows ; row += 3 ) {
9126+ outer_nulls[row] = 1 ;
9127+ }
9128+ ColumnPtr source;
9129+ DataTypePtr source_type;
9130+ ASSERT_TRUE (create_variant_writer_source (VariantWriterInput::V2 , jsons, 0 , false , outer_nulls,
9131+ &source, &source_type)
9132+ .ok ());
9133+
9134+ SegmentFooterPB ref_footer;
9135+ std::string ref_path;
9136+ ASSERT_TRUE (write_extracted_variant_segment (source, source_type, " subcolumn_sparse_ref" ,
9137+ &ref_footer, &ref_path)
9138+ .ok ());
9139+ std::vector<std::string> ref_rows;
9140+ ASSERT_TRUE (read_extracted_variant_rows (ref_footer, ref_path, &ref_rows).ok ());
9141+
9142+ struct ChunkGuard {
9143+ ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (200 ); }
9144+ ~ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (0 ); }
9145+ } guard;
9146+ size_t max_chunks = 0 ;
9147+ SegmentFooterPB chunked_footer;
9148+ std::string chunked_path;
9149+ ASSERT_TRUE (write_extracted_variant_segment (
9150+ source, source_type, " subcolumn_sparse_chunked" , &chunked_footer,
9151+ &chunked_path, 0 , 300 ,
9152+ [&](ColumnWriter* writer) {
9153+ auto * subcolumn = dynamic_cast <VariantSubcolumnWriter*>(writer);
9154+ ASSERT_NE (subcolumn, nullptr );
9155+ max_chunks = std::max (max_chunks, subcolumn->num_chunks_for_test ());
9156+ })
9157+ .ok ());
9158+ EXPECT_GE (max_chunks, 2 );
9159+
9160+ std::vector<std::string> chunked_rows;
9161+ ASSERT_TRUE (read_extracted_variant_rows (chunked_footer, chunked_path, &chunked_rows).ok ());
9162+ EXPECT_EQ (chunked_rows, ref_rows);
9163+ ASSERT_FALSE (chunked_rows.empty ());
9164+ EXPECT_EQ (chunked_rows[0 ], " NULL" );
9165+ EXPECT_EQ (chunked_rows[1 ], ref_rows[1 ]);
9166+ }
9167+
9168+ TEST_F (VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_all_null) {
9169+ // A subcolumn whose every row is null must still append the right number of nulls per chunk.
9170+ init_extracted_variant_schema (11404 );
9171+ constexpr size_t kRows = 2000 ;
9172+ std::vector<std::string> jsons (kRows , " \" v\" " );
9173+ std::vector<UInt8> outer_nulls (kRows , 1 );
9174+ ColumnPtr source;
9175+ DataTypePtr source_type;
9176+ ASSERT_TRUE (create_variant_writer_source (VariantWriterInput::V2 , jsons, 0 , false , outer_nulls,
9177+ &source, &source_type)
9178+ .ok ());
9179+
9180+ SegmentFooterPB ref_footer;
9181+ std::string ref_path;
9182+ ASSERT_TRUE (write_extracted_variant_segment (source, source_type, " subcolumn_null_ref" ,
9183+ &ref_footer, &ref_path)
9184+ .ok ());
9185+ std::vector<std::string> ref_rows;
9186+ ASSERT_TRUE (read_extracted_variant_rows (ref_footer, ref_path, &ref_rows).ok ());
9187+
9188+ struct ChunkGuard {
9189+ ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (100 ); }
9190+ ~ChunkGuard () { VariantSubcolumnWriter::set_chunk_values_for_test (0 ); }
9191+ } guard;
9192+ SegmentFooterPB chunked_footer;
9193+ std::string chunked_path;
9194+ ASSERT_TRUE (write_extracted_variant_segment (source, source_type, " subcolumn_null_chunked" ,
9195+ &chunked_footer, &chunked_path, 0 , 300 )
9196+ .ok ());
9197+ std::vector<std::string> chunked_rows;
9198+ ASSERT_TRUE (read_extracted_variant_rows (chunked_footer, chunked_path, &chunked_rows).ok ());
9199+ EXPECT_EQ (chunked_rows, ref_rows);
9200+ for (const auto & row : chunked_rows) {
9201+ EXPECT_EQ (row, " NULL" );
9202+ }
9203+ }
9204+
9205+ TEST_F (VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_by_bytes) {
9206+ // The byte budget (kChunkBytes) seals a chunk even when the value count is small, e.g. few very
9207+ // wide strings. The chunk must still be flushed independently and read back unchanged.
9208+ init_extracted_variant_schema (11405 );
9209+ constexpr size_t kRows = 3000 ;
9210+ constexpr size_t kBatchRows = 200 ;
9211+ const std::string value (1024 , ' y' );
9212+ const std::string json = " \" " + value + " \" " ;
9213+ std::vector<std::string> jsons (kRows , json);
9214+ ColumnPtr source;
9215+ DataTypePtr source_type;
9216+ ASSERT_TRUE (create_variant_writer_source (VariantWriterInput::V2 , jsons, 0 , false , {}, &source,
9217+ &source_type)
9218+ .ok ());
9219+
9220+ SegmentFooterPB ref_footer;
9221+ std::string ref_path;
9222+ ASSERT_TRUE (write_extracted_variant_segment (source, source_type, " subcolumn_bytes_ref" ,
9223+ &ref_footer, &ref_path)
9224+ .ok ());
9225+ std::vector<std::string> ref_rows;
9226+ ASSERT_TRUE (read_extracted_variant_rows (ref_footer, ref_path, &ref_rows).ok ());
9227+
9228+ struct ChunkGuard {
9229+ ChunkGuard () { VariantSubcolumnWriter::set_chunk_bytes_for_test (256 * 1024 ); }
9230+ ~ChunkGuard () { VariantSubcolumnWriter::set_chunk_bytes_for_test (0 ); }
9231+ } guard;
9232+ size_t max_chunks = 0 ;
9233+ size_t max_chunk_bytes = 0 ;
9234+ SegmentFooterPB chunked_footer;
9235+ std::string chunked_path;
9236+ ASSERT_TRUE (write_extracted_variant_segment (
9237+ source, source_type, " subcolumn_bytes_chunked" , &chunked_footer,
9238+ &chunked_path, 0 , kBatchRows ,
9239+ [&](ColumnWriter* writer) {
9240+ auto * subcolumn = dynamic_cast <VariantSubcolumnWriter*>(writer);
9241+ ASSERT_NE (subcolumn, nullptr );
9242+ max_chunks = std::max (max_chunks, subcolumn->num_chunks_for_test ());
9243+ max_chunk_bytes = std::max (max_chunk_bytes,
9244+ subcolumn->max_chunk_bytes_for_test ());
9245+ })
9246+ .ok ());
9247+ EXPECT_GE (max_chunks, 2 );
9248+ EXPECT_LE (max_chunk_bytes, 256 * 1024 + kBatchRows * 1024 );
9249+
9250+ std::vector<std::string> chunked_rows;
9251+ ASSERT_TRUE (read_extracted_variant_rows (chunked_footer, chunked_path, &chunked_rows).ok ());
9252+ EXPECT_EQ (chunked_rows, ref_rows);
9253+ }
9254+
89769255} // namespace doris
0 commit comments