Skip to content

Commit 32a2651

Browse files
raghav-reglobeclaudemorningman
authored
[fix](iceberg) Fix NPE in COUNT(*) pushdown when snapshot summary omits total-* counters (apache#64648)
## Proposed changes `IcebergScanNode.getCountFromSnapshot()` reads `total-equality-deletes`, `total-position-deletes` and `total-records` from the Iceberg snapshot summary and calls `.equals()` / `Long.parseLong()` directly on the `Map.get()` results. An Iceberg snapshot summary is **not guaranteed** to carry these `total-*` counters — snapshots written by compaction/replace (and some writers) may omit them. When a counter is absent, `SELECT COUNT(*)` throws: ``` java.lang.NullPointerException: Cannot invoke "String.equals(Object)" because the return value of "java.util.Map.get(Object)" is null at org.apache.doris.datasource.iceberg.source.IcebergScanNode.getCountFromSnapshot(IcebergScanNode.java:1154) at org.apache.doris.datasource.iceberg.source.IcebergScanNode.isBatchMode(...) at org.apache.doris.datasource.FileQueryScanNode.createScanRangeLocations(...) ``` `SELECT *` on the same table works (it scans); only `COUNT(*)` fails (it takes the metadata-count shortcut). Reproducible on current master. ### Fix Extract the summary parsing into a pure static `getCountFromSummary(Map<String, String> summary, boolean ignoreDanglingDelete)` that null-checks the counters and **falls back to a normal scan** (`return -1`, the method's existing "cannot push down count" signal) when any required counter is absent. Behaviour is otherwise unchanged. > A similar unguarded access exists in `IcebergUtils` > (`Long.parseLong(summary.get(TOTAL_RECORDS)) - Long.parseLong(summary.get(TOTAL_POSITION_DELETES))`). > Happy to guard it in this PR or a follow-up — let me know your preference. ## Release note Fix a NullPointerException on `SELECT COUNT(*)` over an Iceberg table whose latest snapshot summary omits the `total-*` counters (e.g. snapshots produced by compaction/replace). ## Check List - [x] New feature / bug fix has unit test (`IcebergCountPushDownTest`: missing-counter, no-delete, equality-delete and position-delete cases) - [x] Behaviour-preserving for complete summaries; only adds a null-safe fall-back to scan --------- Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]> Co-authored-by: morningman <[email protected]>
1 parent 98b4a21 commit 32a2651

4 files changed

Lines changed: 127 additions & 21 deletions

File tree

‎be/src/format/table/iceberg_reader_mixin.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,10 +365,14 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper {
365365

366366
template <typename BaseReader>
367367
Status IcebergReaderMixin<BaseReader>::_init_row_filters() {
368-
// COUNT(*) short-circuit
368+
// COUNT(*) short-circuit. A table-level row count of 0 (e.g. an all-deleted table read with
369+
// ignore_iceberg_dangling_delete, where total-records == total-position-deletes) is still a
370+
// valid pushed-down count, so accept >= 0 -- matching FileScanner and the Paimon readers. FE
371+
// sends -1 when there is no table-level count; using > 0 here would drop a genuine 0 into the
372+
// delete-applying path below and never produce the intended CountReader(0).
369373
if (this->_push_down_agg_type == TPushAggOp::type::COUNT &&
370374
this->get_scan_range().table_format_params.__isset.table_level_row_count &&
371-
this->get_scan_range().table_format_params.table_level_row_count > 0) {
375+
this->get_scan_range().table_format_params.table_level_row_count >= 0) {
372376
return Status::OK();
373377
}
374378

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,34 @@ public static int getEffectiveIcebergFormatVersion(Map<String, String> tableProp
216216
}
217217
}
218218

219+
/**
220+
* Decide whether a row count can be read from an Iceberg snapshot summary.
221+
* Returns {@link TableIf#UNKNOWN_ROW_COUNT} when required counters are absent
222+
* or when delete semantics make the summary count unsafe to use.
223+
*/
224+
@VisibleForTesting
225+
public static long getCountFromSummary(Map<String, String> summary, boolean ignoreDanglingDelete) {
226+
String equalityDeletes = summary.get(TOTAL_EQUALITY_DELETES);
227+
String positionDeletes = summary.get(TOTAL_POSITION_DELETES);
228+
String totalRecords = summary.get(TOTAL_RECORDS);
229+
if (equalityDeletes == null || positionDeletes == null || totalRecords == null) {
230+
return TableIf.UNKNOWN_ROW_COUNT;
231+
}
232+
if (!equalityDeletes.equals("0")) {
233+
return TableIf.UNKNOWN_ROW_COUNT;
234+
}
235+
236+
long deleteCount = Long.parseLong(positionDeletes);
237+
if (deleteCount == 0) {
238+
return Long.parseLong(totalRecords);
239+
}
240+
if (ignoreDanglingDelete) {
241+
return Long.parseLong(totalRecords) - deleteCount;
242+
} else {
243+
return TableIf.UNKNOWN_ROW_COUNT;
244+
}
245+
}
246+
219247
public static Expression convertToIcebergExpr(Expr expr, Schema schema) {
220248
if (expr == null) {
221249
return null;
@@ -1175,7 +1203,12 @@ public static long getIcebergRowCount(ExternalTable tbl) {
11751203
return TableIf.UNKNOWN_ROW_COUNT;
11761204
}
11771205
Map<String, String> summary = snapshot.summary();
1178-
long rows = Long.parseLong(summary.get(TOTAL_RECORDS)) - Long.parseLong(summary.get(TOTAL_POSITION_DELETES));
1206+
long rows = getCountFromSummary(summary, true);
1207+
if (rows == TableIf.UNKNOWN_ROW_COUNT) {
1208+
LOG.info("Iceberg table {}.{}.{} row count in summary is unknown, return -1.",
1209+
tbl.getCatalog().getName(), tbl.getDbName(), tbl.getName());
1210+
return TableIf.UNKNOWN_ROW_COUNT;
1211+
}
11791212
LOG.info("Iceberg table {}.{}.{} row count in summary is {}",
11801213
tbl.getCatalog().getName(), tbl.getDbName(), tbl.getName(), rows);
11811214
return rows;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1150,24 +1150,7 @@ public long getCountFromSnapshot() throws UserException {
11501150
return 0;
11511151
}
11521152

1153-
Map<String, String> summary = snapshot.summary();
1154-
if (!summary.get(IcebergUtils.TOTAL_EQUALITY_DELETES).equals("0")) {
1155-
// has equality delete files, can not push down count
1156-
return -1;
1157-
}
1158-
1159-
long deleteCount = Long.parseLong(summary.get(IcebergUtils.TOTAL_POSITION_DELETES));
1160-
if (deleteCount == 0) {
1161-
// no delete files, can push down count directly
1162-
return Long.parseLong(summary.get(IcebergUtils.TOTAL_RECORDS));
1163-
}
1164-
if (sessionVariable.ignoreIcebergDanglingDelete) {
1165-
// has position delete files, if we ignore dangling delete, can push down count
1166-
return Long.parseLong(summary.get(IcebergUtils.TOTAL_RECORDS)) - deleteCount;
1167-
} else {
1168-
// otherwise, can not push down count
1169-
return -1;
1170-
}
1153+
return IcebergUtils.getCountFromSummary(snapshot.summary(), sessionVariable.ignoreIcebergDanglingDelete);
11711154
}
11721155

11731156
@Override
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.iceberg.source;
19+
20+
import org.apache.doris.datasource.iceberg.IcebergUtils;
21+
22+
import com.google.common.collect.ImmutableMap;
23+
import org.junit.jupiter.api.Assertions;
24+
import org.junit.jupiter.api.Test;
25+
26+
import java.util.Collections;
27+
import java.util.Map;
28+
29+
public class IcebergCountPushDownTest {
30+
31+
private static Map<String, String> summary(String equalityDeletes, String positionDeletes, String totalRecords) {
32+
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
33+
if (equalityDeletes != null) {
34+
builder.put(IcebergUtils.TOTAL_EQUALITY_DELETES, equalityDeletes);
35+
}
36+
if (positionDeletes != null) {
37+
builder.put(IcebergUtils.TOTAL_POSITION_DELETES, positionDeletes);
38+
}
39+
if (totalRecords != null) {
40+
builder.put(IcebergUtils.TOTAL_RECORDS, totalRecords);
41+
}
42+
return builder.build();
43+
}
44+
45+
@Test
46+
public void testMissingCounterFallsBackToScan() {
47+
// Snapshots written by compaction/replace (and some writers) may omit
48+
// total-* counters. The pushdown previously NPE'd on the missing key;
49+
// it must now fall back to a normal scan (return -1).
50+
Assertions.assertEquals(-1L, IcebergUtils.getCountFromSummary(summary(null, "0", "100"), false));
51+
Assertions.assertEquals(-1L, IcebergUtils.getCountFromSummary(summary("0", null, "100"), false));
52+
Assertions.assertEquals(-1L, IcebergUtils.getCountFromSummary(summary("0", "0", null), false));
53+
Assertions.assertEquals(-1L, IcebergUtils.getCountFromSummary(Collections.emptyMap(), false));
54+
}
55+
56+
@Test
57+
public void testUtilityMissingCounterReturnsUnknownCount() {
58+
Assertions.assertEquals(-1L, IcebergUtils.getCountFromSummary(summary("0", null, "100"), true));
59+
}
60+
61+
@Test
62+
public void testNoDeletesPushesDownTotalRecords() {
63+
Assertions.assertEquals(100L, IcebergUtils.getCountFromSummary(summary("0", "0", "100"), false));
64+
}
65+
66+
@Test
67+
public void testEqualityDeletesCannotPushDown() {
68+
Assertions.assertEquals(-1L, IcebergUtils.getCountFromSummary(summary("3", "0", "100"), false));
69+
}
70+
71+
@Test
72+
public void testPositionDeletesRespectIgnoreDangling() {
73+
// ignoreDanglingDelete = true -> total-records minus position-deletes
74+
Assertions.assertEquals(90L, IcebergUtils.getCountFromSummary(summary("0", "10", "100"), true));
75+
// ignoreDanglingDelete = false -> cannot push down (fall back to scan)
76+
Assertions.assertEquals(-1L, IcebergUtils.getCountFromSummary(summary("0", "10", "100"), false));
77+
}
78+
79+
@Test
80+
public void testZeroCountWithPositionDeletesIsPushedDown() {
81+
// total-records == position-deletes -> count is 0. With ignore_iceberg_dangling_delete this
82+
// is a valid pushed-down count; FE returns 0 and BE honors it via CountReader(0) (the BE
83+
// table-level guard accepts table_level_row_count >= 0). It must NOT fall back to -1.
84+
Assertions.assertEquals(0L, IcebergUtils.getCountFromSummary(summary("0", "100", "100"), true));
85+
}
86+
}

0 commit comments

Comments
 (0)