Skip to content

Commit 0ff6ed4

Browse files
morningmanclaude
andcommitted
[fix](fluss) Refuse a union read whose log tails will not fit before reading them
BE holds the keys of every log tail it has touched for the whole life of the scan: the lake splits of one bucket arrive spread among the splits of every other bucket, and each of them needs its own tail to suppress with. `fluss.union_read.max_tail_rows` bounds one bucket's tail, so a table with many partitions and many buckets can retain `partitions * buckets` tails at once while every individual one stays far below its own ceiling. That is a query that fails on memory although its input could be streamed. Two changes, both in planning, neither of which needs a byte of the tail: - `fluss.union_read.max_total_tail_rows` bounds the sum. It defaults to ten times the per-bucket ceiling, and a value below the per-bucket one is refused at CREATE rather than silently taken as a smaller per-bucket limit, which would make every later error quote a number nobody configured. - The per-bucket ceiling is now checked here too. The offsets fluss reported while planning already say how many records each tail holds -- `stop - lakeEnd` -- and nothing was reading them. It was left to the readers, so the answer came back only after one of them had read two million rows. Over either ceiling, `auto` gives up the lake half exactly as a truncated tail already does, and the fluss-only read that replaces it caches no tail at all; `required` fails with the number that tripped it. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent a01436d commit 0ff6ed4

6 files changed

Lines changed: 323 additions & 1 deletion

File tree

‎fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussCatalogProperties.java‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,29 @@ public final class FlussCatalogProperties {
9292
/** Default {@link #UNION_READ_MAX_TAIL_ROWS}: two million rows per bucket. */
9393
public static final long DEFAULT_MAX_TAIL_ROWS = 2_000_000L;
9494

95+
/**
96+
* Optional. How many log-tail rows one union-read SCAN may hold across every bucket it touches.
97+
*
98+
* <p>Doris-only, and the scan-wide companion of {@link #UNION_READ_MAX_TAIL_ROWS}: that one bounds any
99+
* single bucket, this one bounds their sum. The two are not the same guard rail. BE caches the keys of
100+
* each tail it reads for the whole life of the scan, because the lake splits of one bucket arrive
101+
* spread among the splits of every other bucket and each of them needs its own tail to suppress with —
102+
* so a table with many partitions and many buckets retains {@code partitions * buckets} tails at once
103+
* while every individual one stays far below its own ceiling.
104+
*
105+
* <p>Checked while planning, from the offsets fluss already reported, so an over-large read is answered
106+
* by a plan that does not need the memory rather than by an allocation failure part-way through it.
107+
*/
108+
public static final String UNION_READ_MAX_TOTAL_TAIL_ROWS =
109+
"fluss.union_read.max_total_tail_rows";
110+
111+
/**
112+
* Default {@link #UNION_READ_MAX_TOTAL_TAIL_ROWS}: ten times {@link #DEFAULT_MAX_TAIL_ROWS}. A scan may
113+
* therefore hold ten maximal tails, or any number of ordinary ones — a healthy tail is the data written
114+
* within one {@code table.datalake.freshness} period, orders of magnitude below its own ceiling.
115+
*/
116+
public static final long DEFAULT_MAX_TOTAL_TAIL_ROWS = 10 * DEFAULT_MAX_TAIL_ROWS;
117+
95118
/**
96119
* Optional. Whether a fluss BINARY/BYTES column reads as Doris VARBINARY instead of STRING.
97120
*
@@ -136,6 +159,10 @@ public String propertyValue() {
136159
description = "guard rail: log-tail rows one bucket may contribute to a union read")
137160
private long maxTailRows = DEFAULT_MAX_TAIL_ROWS;
138161

162+
@ConnectorProperty(names = {UNION_READ_MAX_TOTAL_TAIL_ROWS}, required = false,
163+
description = "guard rail: log-tail rows one union-read scan may hold across all buckets")
164+
private long maxTotalTailRows = DEFAULT_MAX_TOTAL_TAIL_ROWS;
165+
139166
@ConnectorProperty(names = {ENABLE_MAPPING_VARBINARY}, required = false,
140167
description = "map fluss BINARY/BYTES to Doris VARBINARY instead of STRING")
141168
private boolean enableMappingVarbinary;
@@ -170,6 +197,17 @@ public static FlussCatalogProperties of(Map<String, String> properties) {
170197
// ceiling has no way to say so, which is the point.
171198
.check(() -> p.maxTailRows <= 0, "Invalid value '" + p.maxTailRows + "' for property '"
172199
+ UNION_READ_MAX_TAIL_ROWS + "'; expected a positive number of rows")
200+
.check(() -> p.maxTotalTailRows <= 0,
201+
"Invalid value '" + p.maxTotalTailRows + "' for property '"
202+
+ UNION_READ_MAX_TOTAL_TAIL_ROWS
203+
+ "'; expected a positive number of rows")
204+
// A scan-wide ceiling below the per-bucket one cannot describe a scan that passes the
205+
// per-bucket check, so the pair would be unsatisfiable in a way neither value shows on
206+
// its own. Equal is legitimate: it says "one maximal tail, and no more".
207+
.check(() -> p.maxTotalTailRows < p.maxTailRows,
208+
"Property '" + UNION_READ_MAX_TOTAL_TAIL_ROWS + "' (" + p.maxTotalTailRows
209+
+ ") is below '" + UNION_READ_MAX_TAIL_ROWS + "' (" + p.maxTailRows
210+
+ "), so no union read could satisfy both")
173211
.validate();
174212
checkBootstrapServerList(p.bootstrapServers);
175213
// Derived after the checks, so a catalog that fails validation fails on the property it got
@@ -231,7 +269,8 @@ private static Map<String, String> deriveFlussClientConfig(Map<String, String> p
231269

232270
/** Whether {@code key} configures Doris's own behaviour and must not reach the fluss client. */
233271
private static boolean isDorisOnly(String key) {
234-
return UNION_READ_MODE.equals(key) || UNION_READ_MAX_TAIL_ROWS.equals(key);
272+
return UNION_READ_MODE.equals(key) || UNION_READ_MAX_TAIL_ROWS.equals(key)
273+
|| UNION_READ_MAX_TOTAL_TAIL_ROWS.equals(key);
235274
}
236275

237276
private static void checkBootstrapServerList(String value) {
@@ -275,6 +314,12 @@ public long getMaxTailRows() {
275314
return maxTailRows;
276315
}
277316

317+
/** The scan-wide log-tail ceiling for a union read. Always positive, never below {@link
318+
* #getMaxTailRows()}. */
319+
public long getMaxTotalTailRows() {
320+
return maxTotalTailRows;
321+
}
322+
278323
/** The type-mapping switches this catalog declares; both default to off. */
279324
public FlussTypeMapping.Options getTypeMappingOptions() {
280325
return typeMappingOptions;

‎fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,9 @@ public class FlussScanPlanProvider implements ConnectorScanPlanProvider {
187187
/** {@link #degradedReason} when the log no longer holds the tail the lake snapshot stops before. */
188188
private static final String DEGRADED_TAIL_TRUNCATED = "tail-truncated";
189189

190+
/** {@link #degradedReason} when the tails this scan would have to hold are over their ceiling. */
191+
private static final String DEGRADED_TAIL_TOO_LARGE = "tail-too-large";
192+
190193
/** {@link #degradedReason} when a key column's values cannot be compared exactly across the halves. */
191194
private static final String DEGRADED_KEY_TYPE = "key-type";
192195

@@ -310,6 +313,25 @@ private List<ConnectorScanRange> planPrimaryKeyUnion(ConnectorSession session,
310313
return pkRangesFromFlussAlone(states, buckets);
311314
}
312315

316+
// The offsets fluss just reported say exactly how many log records each tail holds, so a read too
317+
// large to hold in memory can be answered by a plan that does not need to hold it -- reading the
318+
// table from fluss alone returns every row without caching a single tail. Doing it here rather
319+
// than at read time also means the ceilings are reported once, from the numbers that tripped them,
320+
// instead of by whichever bucket happened to reach its limit first on some BE.
321+
String tooLarge = firstTailOverBudget(states, buckets);
322+
if (tooLarge != null) {
323+
if (catalogProperties.getUnionReadMode()
324+
== FlussCatalogProperties.UnionReadMode.REQUIRED) {
325+
throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "."
326+
+ handle.getTableName() + "' cannot be read as its lake plus its log: " + tooLarge
327+
+ ". Set '" + FlussCatalogProperties.UNION_READ_MODE + "' to auto or disabled to"
328+
+ " read the table from fluss alone, wait for tiering to move the tail into the"
329+
+ " lake, or raise the ceiling.");
330+
}
331+
degradeToFlussOnly(DEGRADED_TAIL_TOO_LARGE);
332+
return pkRangesFromFlussAlone(states, buckets);
333+
}
334+
313335
List<ConnectorScanRange> ranges = new ArrayList<>();
314336
for (ConnectorScanRange lakeSplit : planLakeRanges(session, union, request)) {
315337
ranges.add(bindTailToLakeSplit(handle, lakeSplit, states));
@@ -425,6 +447,47 @@ private static String firstTruncatedTail(List<PartitionState> states, List<Integ
425447
return null;
426448
}
427449

450+
/**
451+
* The first ceiling a tail of this scan would breach, described for a message, or null when the scan
452+
* fits under both.
453+
*
454+
* <p>Two ceilings, because they answer two different questions. The per-bucket one is what the readers
455+
* enforce while they replay a tail, and reaching it means tiering has stalled for that bucket. The
456+
* scan-wide one has no per-reader equivalent: BE holds the keys of every tail it has touched until the
457+
* scan ends, so a table with many partitions and many buckets can hold far more than any one of them
458+
* would ever report. Summed over the whole scan rather than per BE because planning does not decide
459+
* which BE gets which split, and over-counting a scan that fits is the safe direction.
460+
*/
461+
private String firstTailOverBudget(List<PartitionState> states, List<Integer> buckets) {
462+
long maxTailRows = catalogProperties.getMaxTailRows();
463+
long total = 0;
464+
for (PartitionState state : states) {
465+
for (int bucket : buckets) {
466+
BucketState bucketState = state.buckets.get(bucket);
467+
if (bucketState.lakeEnd == null || bucketState.lakeEnd >= bucketState.stop) {
468+
continue;
469+
}
470+
// Offsets of a bucket's log are consecutive per record, so this IS the record count, and
471+
// it is the same yardstick the readers apply while replaying the range.
472+
long rows = bucketState.stop - bucketState.lakeEnd;
473+
String where = (state.partition.isPartitioned()
474+
? "partition '" + state.partition.getName() + "', " : "") + "bucket " + bucket;
475+
if (rows > maxTailRows) {
476+
return where + " has a log tail of " + rows + " records, over the "
477+
+ maxTailRows + " allowed by '"
478+
+ FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS + "'";
479+
}
480+
total += rows;
481+
}
482+
}
483+
long maxTotal = catalogProperties.getMaxTotalTailRows();
484+
if (total > maxTotal) {
485+
return "its log tails hold " + total + " records in total, over the " + maxTotal
486+
+ " allowed by '" + FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS + "'";
487+
}
488+
return null;
489+
}
490+
428491
/** Every bucket read whole from fluss: the plan {@code disabled} produces, and the safe fallback. */
429492
private static List<ConnectorScanRange> pkRangesFromFlussAlone(List<PartitionState> states,
430493
List<Integer> buckets) {

‎fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussCatalogPropertiesTest.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,12 +151,52 @@ public void theTailCeilingDefaultsToTwoMillionRowsAndMustBePositive() {
151151
bound(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "").getMaxTailRows());
152152
}
153153

154+
/**
155+
* The scan-wide companion of the per-bucket ceiling. It exists because the per-bucket one cannot see
156+
* the case that actually costs the memory: BE keeps the keys of every tail it has touched until the
157+
* scan ends, so many buckets each well inside their own ceiling still add up. A value below the
158+
* per-bucket ceiling describes a scan no read could satisfy, and is rejected rather than silently
159+
* making every union read impossible.
160+
*/
161+
@Test
162+
public void theScanWideTailCeilingDefaultsToTenTailsAndCannotSitBelowThePerBucketOne() {
163+
Assertions.assertEquals(20_000_000L, bound().getMaxTotalTailRows());
164+
// Both, because lowering the scan-wide ceiling under the per-bucket default is itself the
165+
// contradiction this rule refuses -- see the cross-check below.
166+
Assertions.assertEquals(500L,
167+
bound(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "100",
168+
FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, " 500 ")
169+
.getMaxTotalTailRows());
170+
171+
for (String bad : new String[] {"0", "-1", "lots"}) {
172+
IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class,
173+
() -> bound(FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, bad),
174+
"accepted '" + bad + "' as a scan-wide row ceiling");
175+
Assertions.assertTrue(
176+
e.getMessage().contains(FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS),
177+
e.getMessage());
178+
}
179+
180+
IllegalArgumentException below = Assertions.assertThrows(IllegalArgumentException.class,
181+
() -> bound(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "100",
182+
FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, "99"));
183+
Assertions.assertTrue(below.getMessage().contains("no union read could satisfy both"),
184+
below.getMessage());
185+
186+
// Equal is the legitimate edge: "one maximal tail, and no more".
187+
Assertions.assertEquals(100L,
188+
bound(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "100",
189+
FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, "100")
190+
.getMaxTotalTailRows());
191+
}
192+
154193
@Test
155194
public void clientConfigIsThePrefixedPropertiesMinusTheDorisOnlyOnes() {
156195
FlussCatalogProperties p = bound(
157196
"fluss.client.security.protocol", "sasl",
158197
FlussCatalogProperties.UNION_READ_MODE, "required",
159198
FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "10",
199+
FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, "20",
160200
FlussCatalogProperties.ENABLE_MAPPING_VARBINARY, "true",
161201
FlussCatalogProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "true",
162202
"type", "fluss",

‎fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,6 +944,106 @@ public void requiredModeRefusesATailTheLogNoLongerHolds() {
944944
Assertions.assertTrue(e.getMessage().contains("the log now starts at 101"), e.getMessage());
945945
}
946946

947+
/**
948+
* A tail over its own ceiling is caught while planning, from the offsets fluss already reported, rather
949+
* than by whichever BE reached the limit first part-way through reading it. The fluss-only plan that
950+
* replaces it needs no tail in memory at all.
951+
*/
952+
@Test
953+
public void tailOverThePerBucketCeilingGivesUpTheLakeHalf() {
954+
registerPkLakeTable(1);
955+
lakeSnapshotAt(9L, offsets(100L));
956+
kvSnapshots(null, new long[] {4L}, new long[] {10L});
957+
latestOffsets(null, 201L);
958+
earliestOffsets(null, 0L);
959+
FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps,
960+
FlussCatalogProperties.of(catalog(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "100")),
961+
this::lakeSibling);
962+
963+
List<ConnectorScanRange> ranges =
964+
provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList()));
965+
966+
Assertions.assertEquals(1, ranges.size());
967+
assertPkRange(ranges.get(0), 0, 4L, 10L, 201L);
968+
StringBuilder output = new StringBuilder();
969+
provider.appendExplainInfo(output, "", Collections.emptyMap());
970+
Assertions.assertTrue(output.toString().contains("degraded=tail-too-large"), output.toString());
971+
}
972+
973+
/**
974+
* The case the per-bucket ceiling cannot see, and the reason there are two of them: BE holds the keys
975+
* of every tail it has touched until the scan ends, so buckets that are each well inside their own
976+
* ceiling still add up to more than one scan should hold. Two tails of exactly 100 pass the per-bucket
977+
* check of 100 and breach the scan-wide 150 together.
978+
*/
979+
@Test
980+
public void tailsUnderTheirOwnCeilingCanStillBreachTheScanWideOne() {
981+
registerPkLakeTable(2);
982+
lakeSnapshotAt(9L, offsets(100L, 200L));
983+
kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L});
984+
latestOffsets(null, 200L, 300L);
985+
earliestOffsets(null, 0L, 0L);
986+
Map<String, String> properties =
987+
catalog(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "100");
988+
properties.put(FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, "150");
989+
FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps,
990+
FlussCatalogProperties.of(properties), this::lakeSibling);
991+
992+
List<ConnectorScanRange> ranges =
993+
provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList()));
994+
995+
// Fluss-only, and therefore no cached tail on any BE.
996+
Assertions.assertEquals(2, ranges.size());
997+
assertPkRange(ranges.get(0), 0, 4L, 10L, 200L);
998+
assertPkRange(ranges.get(1), 1, 5L, 20L, 300L);
999+
StringBuilder output = new StringBuilder();
1000+
provider.appendExplainInfo(output, "", Collections.emptyMap());
1001+
Assertions.assertTrue(output.toString().contains("degraded=tail-too-large"), output.toString());
1002+
}
1003+
1004+
/** The same two tails are a union read when the scan-wide ceiling has room for both. */
1005+
@Test
1006+
public void tailsThatFitUnderBothCeilingsAreStillMerged() {
1007+
registerPkLakeTable(2);
1008+
lakeSnapshotAt(9L, offsets(100L, 200L));
1009+
kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L});
1010+
latestOffsets(null, 200L, 300L);
1011+
earliestOffsets(null, 0L, 0L);
1012+
Map<String, String> properties =
1013+
catalog(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "100");
1014+
properties.put(FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, "200");
1015+
FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps,
1016+
FlussCatalogProperties.of(properties), this::lakeSibling);
1017+
1018+
provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList()));
1019+
1020+
StringBuilder output = new StringBuilder();
1021+
provider.appendExplainInfo(output, "", Collections.emptyMap());
1022+
Assertions.assertTrue(output.toString().contains("unionRead=yes"), output.toString());
1023+
Assertions.assertFalse(output.toString().contains("degraded="), output.toString());
1024+
}
1025+
1026+
/** Under {@code required} there is nothing to fall back to, so the ceiling is an error with its number. */
1027+
@Test
1028+
public void requiredModeRefusesTailsOverTheScanWideCeiling() {
1029+
registerPkLakeTable(2);
1030+
lakeSnapshotAt(9L, offsets(100L, 200L));
1031+
kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L});
1032+
latestOffsets(null, 200L, 300L);
1033+
earliestOffsets(null, 0L, 0L);
1034+
Map<String, String> properties =
1035+
catalog(FlussCatalogProperties.UNION_READ_MODE, "required");
1036+
properties.put(FlussCatalogProperties.UNION_READ_MAX_TAIL_ROWS, "100");
1037+
properties.put(FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS, "150");
1038+
1039+
DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class,
1040+
() -> plan(PK_TABLE, properties));
1041+
Assertions.assertTrue(e.getMessage().contains("200 records in total"), e.getMessage());
1042+
Assertions.assertTrue(
1043+
e.getMessage().contains(FlussCatalogProperties.UNION_READ_MAX_TOTAL_TAIL_ROWS),
1044+
e.getMessage());
1045+
}
1046+
9471047
/**
9481048
* A key column Doris cannot compare exactly is a permanent property of the table, so it is settled
9491049
* before anything is asked of the lake — that is what lets the answer be the same at plan-translation

‎regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,4 +120,31 @@ suite("test_fluss_catalog", "p0,external") {
120120
"""
121121
exception "expected one of auto, required, disabled"
122122
}
123+
124+
// The two union-read ceilings bound different things -- one bucket's tail, and the sum a scan
125+
// holds across every bucket at once -- so a scan-wide value below the per-bucket one describes a
126+
// read nothing could satisfy. Refused here rather than silently taking the smaller of the two,
127+
// which would make every later error quote a number nobody configured.
128+
test {
129+
sql """
130+
create catalog test_fluss_ceiling_conflict properties (
131+
"type" = "fluss",
132+
"fluss.bootstrap.servers" = "${bootstrapServers}",
133+
"fluss.union_read.max_tail_rows" = "100",
134+
"fluss.union_read.max_total_tail_rows" = "99"
135+
);
136+
"""
137+
exception "no union read could satisfy both"
138+
}
139+
140+
test {
141+
sql """
142+
create catalog test_fluss_bad_total_ceiling properties (
143+
"type" = "fluss",
144+
"fluss.bootstrap.servers" = "${bootstrapServers}",
145+
"fluss.union_read.max_total_tail_rows" = "0"
146+
);
147+
"""
148+
exception "fluss.union_read.max_total_tail_rows"
149+
}
123150
}

0 commit comments

Comments
 (0)