Skip to content

Commit 11a2544

Browse files
coderex2522dongjoon-hyun
authored andcommitted
ORC-1170: [C++] Optimize the RowReader::seekToRow function in the case of seeking within the same stripe
### What changes were proposed in this pull request? The patch will optimize the seekToRow function. The RowReader::seekToRow() function always call startNextStripe() to seek to the beginning of target stripe. However, when PPD is enabled many calls are still staying in the same stripe which results in many unnecessary calls. ### Why are the changes needed? This PR improves the seekToRow function call. I pick a 259MB file of the TPCH lineitem table which contains 7200030 records. I use a modified orc-scan tool that calls the RowReader::seekToRow function once before each call the RowReader:: next() function to test this patch. The results are shown below: || original | optimized | speedup | |:-----|:----:|:----:|:----:| |Elapsed Time(s)|43.69|27.13|1.61| ### How was this patch tested? The original UTs(TestPredicatePushdown, DictionaryEncoding::multipleStripesWithIndex, DictionaryEncoding::multipleStripesWithoutIndex) will cover this patch. Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent c106c80 commit 11a2544

3 files changed

Lines changed: 40 additions & 15 deletions

File tree

‎c++/include/orc/MemoryPool.hh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ namespace orc {
7272
return currentCapacity;
7373
}
7474

75+
const T& operator[](uint64_t i) const {
76+
return buf[i];
77+
}
78+
7579
T& operator[](uint64_t i) {
7680
return buf[i];
7781
}

‎c++/src/Reader.cc‎

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,7 @@ namespace orc {
282282
}
283283
}
284284
firstStripe = currentStripe;
285+
processingStripe = lastStripe;
285286

286287
if (currentStripe == 0) {
287288
previousRow = (std::numeric_limits<uint64_t>::max)();
@@ -391,27 +392,40 @@ namespace orc {
391392
return;
392393
}
393394

394-
currentStripe = seekToStripe;
395-
currentRowInStripe = rowNumber - firstRowOfStripe[currentStripe];
396395
previousRow = rowNumber;
397-
startNextStripe();
396+
auto rowIndexStride = footer->rowindexstride();
397+
if (!isCurrentStripeInited()
398+
|| currentStripe != seekToStripe
399+
|| rowIndexStride == 0
400+
|| currentStripeInfo.indexlength() == 0) {
401+
// current stripe is not initialized or
402+
// target stripe is not current stripe or
403+
// current stripe doesn't have row indexes
404+
currentStripe = seekToStripe;
405+
currentRowInStripe = rowNumber - firstRowOfStripe[currentStripe];
406+
startNextStripe();
407+
if (currentStripe >= lastStripe) {
408+
return;
409+
}
410+
} else {
411+
currentRowInStripe = rowNumber - firstRowOfStripe[currentStripe];
412+
if (sargsApplier) {
413+
// advance to selected row group if predicate pushdown is enabled
414+
currentRowInStripe = advanceToNextRowGroup(currentRowInStripe,
415+
rowsInCurrentStripe,
416+
footer->rowindexstride(),
417+
sargsApplier->getNextSkippedRows());
418+
}
419+
}
398420

399421
uint64_t rowsToSkip = currentRowInStripe;
400-
auto rowIndexStride = footer->rowindexstride();
401422
// seek to the target row group if row indexes exists
402423
if (rowIndexStride > 0 && currentStripeInfo.indexlength() > 0) {
403-
// when predicate push down is enabled, above call to startNextStripe()
404-
// will move current row to 1st matching row group; here we only need
405-
// to deal with the case when PPD is not enabled.
406-
if (!sargsApplier) {
407-
if (rowIndexes.empty()) {
408-
loadStripeIndex();
409-
}
410-
auto rowGroupId = static_cast<uint32_t>(rowsToSkip / rowIndexStride);
411-
if (rowGroupId != 0) {
412-
seekToRowGroup(rowGroupId);
413-
}
424+
if (rowIndexes.empty()) {
425+
loadStripeIndex();
414426
}
427+
// TODO(ORC-1175): process the failures of loadStripeIndex() call
428+
seekToRowGroup(static_cast<uint32_t>(rowsToSkip / rowIndexStride));
415429
// skip leading rows in the target row group
416430
rowsToSkip %= rowIndexStride;
417431
}
@@ -1075,6 +1089,7 @@ namespace orc {
10751089
}
10761090
currentStripeFooter = getStripeFooter(currentStripeInfo, *contents.get());
10771091
rowsInCurrentStripe = currentStripeInfo.numberofrows();
1092+
processingStripe = currentStripe;
10781093

10791094
if (sargsApplier) {
10801095
bool isStripeNeeded = true;

‎c++/src/Reader.hh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ namespace orc {
146146
uint64_t firstStripe;
147147
uint64_t currentStripe;
148148
uint64_t lastStripe; // the stripe AFTER the last one
149+
uint64_t processingStripe;
149150
uint64_t currentRowInStripe;
150151
uint64_t rowsInCurrentStripe;
151152
proto::StripeInformation currentStripeInfo;
@@ -187,6 +188,11 @@ namespace orc {
187188
friend class TestRowReader_advanceToNextRowGroup_Test;
188189
friend class TestRowReader_computeBatchSize_Test;
189190

191+
// whether the current stripe is initialized
192+
inline bool isCurrentStripeInited() const {
193+
return currentStripe == processingStripe;
194+
}
195+
190196
/**
191197
* Seek to the start of a row group in the current stripe
192198
* @param rowGroupEntryId the row group id to seek to

0 commit comments

Comments
 (0)