Skip to content

Commit d3a576d

Browse files
authored
GH-37002: [C++][Parquet] Add api to get RecordReader from RowGroupReader (#37003)
Currently we only can get a ColumnReader for a column from RowGroupReader. We need an API to return a RecordReader for the column. Moved ComputeLevelInfo from column_writer to level_conversion so that it can be shared between column_writer and file_reader. * Closes: #37002 Lead-authored-by: Fatemah Panahi <[email protected]> Co-authored-by: Fatemah Panahi <[email protected]> Signed-off-by: Gang Wu <[email protected]>
1 parent b3b958e commit d3a576d

6 files changed

Lines changed: 89 additions & 18 deletions

File tree

‎cpp/src/parquet/column_writer.cc‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -140,23 +140,6 @@ struct ValueBufferSlicer {
140140
MemoryPool* pool_;
141141
};
142142

143-
internal::LevelInfo ComputeLevelInfo(const ColumnDescriptor* descr) {
144-
internal::LevelInfo level_info;
145-
level_info.def_level = descr->max_definition_level();
146-
level_info.rep_level = descr->max_repetition_level();
147-
148-
int16_t min_spaced_def_level = descr->max_definition_level();
149-
const ::parquet::schema::Node* node = descr->schema_node().get();
150-
while (node != nullptr && !node->is_repeated()) {
151-
if (node->is_optional()) {
152-
min_spaced_def_level--;
153-
}
154-
node = node->parent();
155-
}
156-
level_info.repeated_ancestor_def_level = min_spaced_def_level;
157-
return level_info;
158-
}
159-
160143
template <class T>
161144
inline const T* AddIfNotNull(const T* base, int64_t offset) {
162145
if (base != nullptr) {
@@ -738,7 +721,7 @@ class ColumnWriterImpl {
738721
Encoding::type encoding, const WriterProperties* properties)
739722
: metadata_(metadata),
740723
descr_(metadata->descr()),
741-
level_info_(ComputeLevelInfo(metadata->descr())),
724+
level_info_(internal::LevelInfo::ComputeLevelInfo(metadata->descr())),
742725
pager_(std::move(pager)),
743726
has_dictionary_(use_dictionary),
744727
encoding_(encoding),

‎cpp/src/parquet/file_reader.cc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,26 @@ std::shared_ptr<ColumnReader> RowGroupReader::Column(int i) {
8282
const_cast<ReaderProperties*>(contents_->properties())->memory_pool());
8383
}
8484

85+
std::shared_ptr<internal::RecordReader> RowGroupReader::RecordReader(int i) {
86+
if (i >= metadata()->num_columns()) {
87+
std::stringstream ss;
88+
ss << "Trying to read column index " << i << " but row group metadata has only "
89+
<< metadata()->num_columns() << " columns";
90+
throw ParquetException(ss.str());
91+
}
92+
const ColumnDescriptor* descr = metadata()->schema()->Column(i);
93+
94+
std::unique_ptr<PageReader> page_reader = contents_->GetColumnPageReader(i);
95+
96+
internal::LevelInfo level_info = internal::LevelInfo::ComputeLevelInfo(descr);
97+
98+
auto reader = internal::RecordReader::Make(
99+
descr, level_info, contents_->properties()->memory_pool(),
100+
/* read_dictionary = */ false, contents_->properties()->read_dense_for_nullable());
101+
reader->SetPageReader(std::move(page_reader));
102+
return reader;
103+
}
104+
85105
std::shared_ptr<ColumnReader> RowGroupReader::ColumnWithExposeEncoding(
86106
int i, ExposedEncoding encoding_to_expose) {
87107
std::shared_ptr<ColumnReader> reader = Column(i);

‎cpp/src/parquet/file_reader.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ class BloomFilterReader;
3737
class PageReader;
3838
class RowGroupMetaData;
3939

40+
namespace internal {
41+
class RecordReader;
42+
}
43+
4044
class PARQUET_EXPORT RowGroupReader {
4145
public:
4246
// Forward declare a virtual class 'Contents' to aid dependency injection and more
@@ -58,6 +62,10 @@ class PARQUET_EXPORT RowGroupReader {
5862
// column. Ownership is shared with the RowGroupReader.
5963
std::shared_ptr<ColumnReader> Column(int i);
6064

65+
// EXPERIMENTAL: Construct a RecordReader for the indicated column of the row group.
66+
// Ownership is shared with the RowGroupReader.
67+
std::shared_ptr<internal::RecordReader> RecordReader(int i);
68+
6169
// Construct a ColumnReader, trying to enable exposed encoding.
6270
//
6371
// For dictionary encoding, currently we only support column chunks that are fully

‎cpp/src/parquet/level_conversion.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,24 @@ struct PARQUET_EXPORT LevelInfo {
121121
return last_repeated_ancestor;
122122
}
123123

124+
// Calculates and returns LevelInfo for a column descriptor.
125+
static LevelInfo ComputeLevelInfo(const ColumnDescriptor* descr) {
126+
LevelInfo level_info;
127+
level_info.def_level = descr->max_definition_level();
128+
level_info.rep_level = descr->max_repetition_level();
129+
130+
int16_t min_spaced_def_level = descr->max_definition_level();
131+
const ::parquet::schema::Node* node = descr->schema_node().get();
132+
while (node && !node->is_repeated()) {
133+
if (node->is_optional()) {
134+
min_spaced_def_level--;
135+
}
136+
node = node->parent();
137+
}
138+
level_info.repeated_ancestor_def_level = min_spaced_def_level;
139+
return level_info;
140+
}
141+
124142
friend std::ostream& operator<<(std::ostream& os, const LevelInfo& levels) {
125143
// This print method is to silence valgrind issues. What's printed
126144
// is not important because all asserts happen directly on

‎cpp/src/parquet/properties.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ class PARQUET_EXPORT ReaderProperties {
7979
/// Disable buffered stream reading.
8080
void disable_buffered_stream() { buffered_stream_enabled_ = false; }
8181

82+
bool read_dense_for_nullable() const { return read_dense_for_nullable_; }
83+
void enable_read_dense_for_nullable() { read_dense_for_nullable_ = true; }
84+
void disable_read_dense_for_nullable() { read_dense_for_nullable_ = false; }
85+
8286
/// Return the size of the buffered stream buffer.
8387
int64_t buffer_size() const { return buffer_size_; }
8488
/// Set the size of the buffered stream buffer in bytes.
@@ -123,6 +127,8 @@ class PARQUET_EXPORT ReaderProperties {
123127
int32_t thrift_container_size_limit_ = kDefaultThriftContainerSizeLimit;
124128
bool buffered_stream_enabled_ = false;
125129
bool page_checksum_verification_ = false;
130+
// Used with a RecordReader.
131+
bool read_dense_for_nullable_ = false;
126132
std::shared_ptr<FileDecryptionProperties> file_decryption_properties_;
127133
};
128134

‎cpp/src/parquet/reader_test.cc‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,42 @@ TEST_F(TestAllTypesPlain, ColumnSelectionOutOfRange) {
502502
ASSERT_THROW(printer2.DebugPrint(ss, columns), ParquetException);
503503
}
504504

505+
// Tests that read_dense_for_nullable is passed down to the record
506+
// reader. The functionality of read_dense_for_nullable is tested
507+
// elsewhere.
508+
TEST(TestFileReader, RecordReaderReadDenseForNullable) {
509+
// We test the default which is false, and also test enabling and disabling
510+
// read_dense_for_nullable.
511+
std::vector<ReaderProperties> reader_properties(3);
512+
reader_properties[1].enable_read_dense_for_nullable();
513+
reader_properties[2].disable_read_dense_for_nullable();
514+
for (const auto& reader_props : reader_properties) {
515+
std::unique_ptr<ParquetFileReader> file_reader = ParquetFileReader::OpenFile(
516+
alltypes_plain(), /* memory_map = */ false, reader_props);
517+
std::shared_ptr<RowGroupReader> group = file_reader->RowGroup(0);
518+
std::shared_ptr<internal::RecordReader> col_record_reader = group->RecordReader(0);
519+
ASSERT_EQ(reader_props.read_dense_for_nullable(),
520+
col_record_reader->read_dense_for_nullable());
521+
}
522+
}
523+
524+
// Tests getting a record reader from a row group reader.
525+
TEST(TestFileReader, GetRecordReader) {
526+
ReaderProperties reader_props;
527+
std::unique_ptr<ParquetFileReader> file_reader = ParquetFileReader::OpenFile(
528+
alltypes_plain(), /* memory_map = */ false, reader_props);
529+
std::shared_ptr<RowGroupReader> group = file_reader->RowGroup(0);
530+
531+
std::shared_ptr<internal::RecordReader> col_record_reader_ = group->RecordReader(0);
532+
533+
ASSERT_TRUE(col_record_reader_->HasMoreData());
534+
auto records_read = col_record_reader_->ReadRecords(4);
535+
ASSERT_EQ(records_read, 4);
536+
ASSERT_EQ(4, col_record_reader_->values_written());
537+
ASSERT_EQ(4, col_record_reader_->levels_position());
538+
ASSERT_EQ(8, col_record_reader_->levels_written());
539+
}
540+
505541
class TestLocalFile : public ::testing::Test {
506542
public:
507543
void SetUp() {

0 commit comments

Comments
 (0)