Skip to content

Commit 598756e

Browse files
chenjunweichenjunwei
authored andcommitted
[fix](fe) Read connector statistics at the query snapshot
### What problem does this PR solve? Issue Number: close apache#69884 Related PR: apache#69904 Problem Summary: Ordinary external-table queries only consulted the table-keyed row-count cache, which could keep a cold estimate after versioned connector statistics completed. Prefer statistics at the statement's pinned snapshot for ordinary and versioned queries, reuse the existing size/row-width fallback, and forward the pinned snapshot through the Hive sibling gateway. This covers STAT-02. ### Release note External-table optimizer estimates can use completed statistics for the query's pinned snapshot without waiting for the table-only row-count cache to refresh. ### Check List (For Author) - Test: Full FE build and Checkstyle; 114 FE core and 267 Hive tests; focused compatibility run including 22 Paimon and 7 Iceberg tests; seven Delta p0 suites on the combined change, including new filesystem/HMS/Hive cardinality coverage. - Behavior changed: Yes, ordinary pinned queries prefer snapshot statistics. - Does this need documentation: Yes, Delta connector README and prepared EN/ZH website patch are updated in the accompanying statistics change; website PR pending.
1 parent ef4b7b2 commit 598756e

4 files changed

Lines changed: 114 additions & 40 deletions

File tree

‎fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,15 @@ private static String parseTrinoViewDefinition(String originalText) {
872872

873873
// ========== ConnectorStatisticsOps ==========
874874

875+
@Override
876+
public Optional<ConnectorTableStatistics> getTableStatistics(ConnectorSession session, ConnectorTableHandle handle,
877+
ConnectorMvccSnapshot snapshot) {
878+
if (!(handle instanceof HiveTableHandle)) {
879+
return siblingMetadata(session, handle).getTableStatistics(session, handle, snapshot);
880+
}
881+
return getTableStatistics(session, handle);
882+
}
883+
875884
/**
876885
* Table-level statistics for a hive table, a port of legacy {@code StatisticsUtil.getHiveRowCount} +
877886
* {@code getTotalSizeFromHMS} restricted to the two RAW metastore facts (no Doris-type math — the

‎fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,9 @@ public void everyPerHandleMethodForwardsAForeignHandleToTheSibling() {
239239
md.getTableSchema(session, foreignHandle);
240240
md.getColumnHandles(session, foreignHandle);
241241
md.getTableStatistics(session, foreignHandle);
242+
ConnectorMvccSnapshot statisticsPin = ConnectorMvccSnapshot.builder().snapshotId(123).build();
243+
Assertions.assertEquals(123, md.getTableStatistics(session, foreignHandle, statisticsPin)
244+
.orElseThrow().getRowCount());
242245
md.getColumnStatistics(session, foreignHandle, "c");
243246
long size = md.estimateDataSizeByListingFiles(session, foreignHandle);
244247
Optional<FilterApplicationResult<ConnectorTableHandle>> filter = md.applyFilter(session, foreignHandle, null);
@@ -725,7 +728,8 @@ private static final class RecordingSiblingMetadata implements ConnectorMetadata
725728
// The exact set + order of forwarded methods the foreign-handle test drives (a Rule-9 completeness lock:
726729
// dropping a guard, or adding one that should not forward, changes this list and fails the test).
727730
static final List<String> EXPECTED_METHODS = Collections.unmodifiableList(Arrays.asList(
728-
"getTableSchema", "getColumnHandles", "getTableStatistics", "getColumnStatistics",
731+
"getTableSchema", "getColumnHandles", "getTableStatistics", "getTableStatisticsAtSnapshot",
732+
"getColumnStatistics",
729733
"estimateDataSizeByListingFiles",
730734
"applyFilter", "listPartitionNames", "listPartitions",
731735
"beginQuerySnapshot", "getTableFreshness", "getPartitionFreshnessMillis",
@@ -779,6 +783,13 @@ public Optional<ConnectorTableStatistics> getTableStatistics(ConnectorSession se
779783
return Optional.of(new ConnectorTableStatistics(1L, 2L));
780784
}
781785

786+
@Override
787+
public Optional<ConnectorTableStatistics> getTableStatistics(ConnectorSession session,
788+
ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) {
789+
calls.add("getTableStatisticsAtSnapshot");
790+
return Optional.of(new ConnectorTableStatistics(snapshot.getSnapshotId(), 2));
791+
}
792+
782793
@Override
783794
public Optional<ConnectorColumnStatistics> getColumnStatistics(ConnectorSession session,
784795
ConnectorTableHandle handle, String columnName) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java‎

Lines changed: 30 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1460,26 +1460,9 @@ private long fetchRowCount(ConnectorSession session, ConnectorMetadata metadata)
14601460

14611461
Optional<ConnectorTableStatistics> statsOpt = metadata.getTableStatistics(session, handleOpt.get());
14621462
if (statsOpt.isPresent()) {
1463-
ConnectorTableStatistics stats = statsOpt.get();
1464-
if (stats.getRowCount() >= 0) {
1465-
return stats.getRowCount();
1466-
}
1467-
// The connector surfaced an on-disk data size but no exact row count (e.g. a hive table with
1468-
// totalSize set but no numRows). Estimate the cardinality as dataSize / <average Doris row
1469-
// width> — the Doris-type-dependent division the connector cannot perform (it must not import
1470-
// fe-type). Connector-agnostic: every other connector reports dataSize -1, so this branch is
1471-
// inert for them. Mirrors legacy StatisticsUtil.getHiveRowCount's totalSize/estimatedRowSize
1472-
// path (row width summed over the FULL schema, partition columns included, exactly as legacy).
1473-
// A quotient that truncates to 0 is NOT a valid "empty table" answer — legacy collapsed 0 ->
1474-
// UNKNOWN and fell through to the file-list estimate below, so only a positive quotient returns.
1475-
if (stats.getDataSize() > 0) {
1476-
long rowWidth = estimatedRowWidth(false);
1477-
if (rowWidth > 0) {
1478-
long rows = stats.getDataSize() / rowWidth;
1479-
if (rows > 0) {
1480-
return rows;
1481-
}
1482-
}
1463+
long rows = rowCountFromStatistics(statsOpt.get());
1464+
if (rows != UNKNOWN_ROW_COUNT) {
1465+
return rows;
14831466
}
14841467
}
14851468

@@ -1507,16 +1490,12 @@ private long fetchRowCount(ConnectorSession session, ConnectorMetadata metadata)
15071490

15081491
@Override
15091492
public long getRowCount() {
1510-
// Time-travel row count: the shared cross-statement row-count cache is keyed by table only and
1511-
// computes at the LATEST snapshot, so a FOR VERSION/TIME AS OF (or @branch/@tag) read would get the
1512-
// latest cardinality while its scan reads the pinned snapshot -> skewed CBO estimate (estimate-only;
1513-
// results stay correct). When THIS statement pins a genuine versioned snapshot for this table, compute
1514-
// the row count directly AT that snapshot, bypassing the latest-keyed shared cache (a historical count
1515-
// is not worth caching cross-statement). A plain/latest read has no versioned pin and keeps the cached
1516-
// path unchanged; a call with no statement context (e.g. background stats) also keeps it.
1493+
// Prefer the statement's snapshot for ordinary reads as well as time travel. The shared
1494+
// table-only cache can retain an older version or an initial size estimate after a connector's
1495+
// asynchronous version-scoped statistics have completed. Connectors own this fast path's cache.
15171496
ConnectContext ctx = ConnectContext.get();
15181497
if (ctx != null && ctx.getStatementContext() != null) {
1519-
Optional<MvccSnapshot> versioned = ctx.getStatementContext().getVersionedSnapshot(this);
1498+
Optional<MvccSnapshot> versioned = ctx.getStatementContext().getSnapshot(this);
15201499
if (versioned.isPresent() && versioned.get() instanceof PluginDrivenMvccSnapshot) {
15211500
long rows = fetchRowCountAtSnapshot(
15221501
((PluginDrivenMvccSnapshot) versioned.get()).getConnectorSnapshot());
@@ -1530,13 +1509,10 @@ public long getRowCount() {
15301509
}
15311510

15321511
/**
1533-
* Computes the row count AT a pinned snapshot for a time-travel read: mirrors the exact-count branch of
1534-
* {@link #fetchRowCount()} but threads the snapshot into the 3-arg {@code getTableStatistics}. Runs in the
1535-
* query thread (not the background cache loader, which has no statement context), so it is deliberately
1536-
* NOT cached &mdash; the handful of versioned reads per statement is cheap. Returns
1537-
* {@link #UNKNOWN_ROW_COUNT} when the connector cannot serve an exact count at the snapshot. Ordinary
1538-
* tables may fall back to the latest estimate; system relations must preserve UNKNOWN to avoid snapshot
1539-
* skew, so the decision belongs to the caller.
1512+
* Retrieves statistics at the statement's pinned snapshot, including an on-disk size estimate when
1513+
* a row count is not yet available. This query-thread path must remain cheap: connectors cache or
1514+
* asynchronously load expensive statistics. Unknown ordinary-table statistics retain the existing
1515+
* latest-estimate fallback; system relations preserve UNKNOWN to avoid snapshot skew.
15401516
*/
15411517
protected long fetchRowCountAtSnapshot(ConnectorMvccSnapshot snapshot) {
15421518
makeSureInitialized();
@@ -1551,15 +1527,30 @@ protected long fetchRowCountAtSnapshot(ConnectorMvccSnapshot snapshot) {
15511527
}
15521528
Optional<ConnectorTableStatistics> statsOpt =
15531529
metadata.getTableStatistics(session, handleOpt.get(), snapshot);
1554-
if (statsOpt.isPresent() && statsOpt.get().getRowCount() >= 0) {
1555-
return statsOpt.get().getRowCount();
1556-
}
1557-
return UNKNOWN_ROW_COUNT;
1530+
return statsOpt.map(this::rowCountFromStatistics).orElse(UNKNOWN_ROW_COUNT);
15581531
} finally {
15591532
session.getStatementScope().closeAll();
15601533
}
15611534
}
15621535

1536+
private long rowCountFromStatistics(ConnectorTableStatistics stats) {
1537+
if (stats.getRowCount() >= 0) {
1538+
return stats.getRowCount();
1539+
}
1540+
// Preserve the Hive size/slot-width estimate and use it for snapshot-aware connectors too.
1541+
// Compressed bytes only provide a heuristic: a quotient of zero does not prove an empty table.
1542+
if (stats.getDataSize() > 0) {
1543+
long rowWidth = estimatedRowWidth(false);
1544+
if (rowWidth > 0) {
1545+
long rows = stats.getDataSize() / rowWidth;
1546+
if (rows > 0) {
1547+
return rows;
1548+
}
1549+
}
1550+
}
1551+
return UNKNOWN_ROW_COUNT;
1552+
}
1553+
15631554
/**
15641555
* Sum of Doris slot sizes over the full schema (or over the non-partition columns when
15651556
* {@code excludePartitionColumns}) — the average uncompressed row width used to turn a connector-reported

‎fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableRowCountTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,13 @@
2626
import org.apache.doris.connector.spi.ConnectorStatementScope;
2727
import org.apache.doris.connector.spi.ConnectorTableStatistics;
2828
import org.apache.doris.connector.spi.handle.ConnectorTableHandle;
29+
import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot;
2930
import org.apache.doris.datasource.ExternalDatabase;
3031
import org.apache.doris.datasource.SchemaCacheValue;
3132
import org.apache.doris.datasource.SessionContext;
33+
import org.apache.doris.datasource.mvcc.PluginDrivenMvccSnapshot;
34+
import org.apache.doris.nereids.StatementContext;
35+
import org.apache.doris.qe.ConnectContext;
3236
import org.apache.doris.qe.GlobalVariable;
3337

3438
import org.junit.jupiter.api.Assertions;
@@ -41,6 +45,8 @@
4145
import java.util.List;
4246
import java.util.Map;
4347
import java.util.Optional;
48+
import java.util.concurrent.atomic.AtomicReference;
49+
import java.util.function.Function;
4450

4551
/**
4652
* Tests {@link PluginDrivenExternalTable#fetchRowCount}'s two-layer statistics consumption (§4.2 read-side
@@ -116,6 +122,55 @@ public void unresolvableHandleYieldsUnknown() {
116122
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, table.fetchRowCount());
117123
}
118124

125+
@Test
126+
public void ordinarySnapshotUsesSizeFallbackThenCompletedVersionStatistics() {
127+
AtomicReference<ConnectorTableStatistics> result =
128+
new AtomicReference<>(new ConnectorTableStatistics(-1, 400));
129+
PluginDrivenExternalTable table = tableReturning(Optional.of(new ConnectorTableStatistics(9999, 9999)),
130+
Collections.singletonList(intCol("v")), snapshot -> Optional.of(result.get()));
131+
ConnectorMvccSnapshot pin = ConnectorMvccSnapshot.builder().snapshotId(7).build();
132+
ConnectContext previous = ConnectContext.get();
133+
ConnectContext context = new ConnectContext();
134+
StatementContext statement = Mockito.mock(StatementContext.class);
135+
Mockito.when(statement.getSnapshot(table)).thenReturn(Optional.of(new PluginDrivenMvccSnapshot(
136+
pin, Collections.emptyMap(), Collections.emptyMap())));
137+
context.setStatementContext(statement);
138+
context.setThreadLocalInfo();
139+
try {
140+
Assertions.assertEquals(100, table.getRowCount());
141+
result.set(new ConnectorTableStatistics(37, 400));
142+
Assertions.assertEquals(37, table.getRowCount(), "completed count must replace the temporary size estimate");
143+
Mockito.verify(statement, Mockito.never()).getVersionedSnapshot(table);
144+
} finally {
145+
if (previous == null) {
146+
ConnectContext.remove();
147+
} else {
148+
previous.setThreadLocalInfo();
149+
}
150+
}
151+
}
152+
153+
@Test
154+
public void pinnedStatisticsKeepVersionsAndEmptyTablesDistinct() {
155+
PluginDrivenExternalTable table = tableReturning(Optional.of(new ConnectorTableStatistics(9999, 9999)),
156+
Collections.singletonList(intCol("v")), snapshot -> Optional.of(
157+
new ConnectorTableStatistics(snapshot.getSnapshotId() == 7 ? 37 : 0, 400)));
158+
Assertions.assertEquals(37, table.fetchRowCountAtSnapshot(
159+
ConnectorMvccSnapshot.builder().snapshotId(7).build()));
160+
Assertions.assertEquals(0, table.fetchRowCountAtSnapshot(
161+
ConnectorMvccSnapshot.builder().snapshotId(8).build()));
162+
}
163+
164+
@Test
165+
public void tinyOrAbsentSnapshotSizeIsUnknownRatherThanEmpty() {
166+
PluginDrivenExternalTable table = tableReturning(Optional.empty(), Collections.singletonList(bigintCol("v")),
167+
snapshot -> Optional.of(new ConnectorTableStatistics(-1, snapshot.getSnapshotId())));
168+
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, table.fetchRowCountAtSnapshot(
169+
ConnectorMvccSnapshot.builder().snapshotId(1).build()));
170+
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, table.fetchRowCountAtSnapshot(
171+
ConnectorMvccSnapshot.builder().snapshotId(0).build()));
172+
}
173+
119174
// ==================== layer 3: file-list data-size estimate ====================
120175

121176
@Test
@@ -264,6 +319,11 @@ public Optional<SchemaCacheValue> getSchemaCacheValue() {
264319
*/
265320
private static PluginDrivenExternalTable tableReturning(
266321
Optional<ConnectorTableStatistics> stats, List<Column> schema) {
322+
return tableReturning(stats, schema, snapshot -> Optional.empty());
323+
}
324+
325+
private static PluginDrivenExternalTable tableReturning(Optional<ConnectorTableStatistics> stats,
326+
List<Column> schema, Function<ConnectorMvccSnapshot, Optional<ConnectorTableStatistics>> atSnapshot) {
267327
ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class);
268328
ConnectorSession session = Mockito.mock(ConnectorSession.class);
269329
Mockito.when(session.getStatementScope()).thenReturn(ConnectorStatementScope.NONE);
@@ -275,6 +335,9 @@ private static PluginDrivenExternalTable tableReturning(
275335
Mockito.when(metadata.getTableHandle(session, "REMOTE_DB", "REMOTE_TBL"))
276336
.thenReturn(Optional.of(handle));
277337
Mockito.when(metadata.getTableStatistics(session, handle)).thenReturn(stats);
338+
Mockito.when(metadata.getTableStatistics(Mockito.eq(session), Mockito.eq(handle),
339+
Mockito.any(ConnectorMvccSnapshot.class)))
340+
.thenAnswer(invocation -> atSnapshot.apply(invocation.getArgument(2)));
278341
}
279342
TestablePluginCatalog catalog = new TestablePluginCatalog(metadata, session);
280343

0 commit comments

Comments
 (0)