Skip to content

Commit 2e7f79e

Browse files
authored
[fix](iceberg) Map branch retention times correctly (apache#69449)
### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: `ALTER TABLE ... CREATE BRANCH ... RETAIN <time>` was mapped to `max-snapshot-age-ms` instead of `max-ref-age-ms`. When both `RETAIN` and `WITH SNAPSHOT RETENTION ... <time>` were specified, the two time values were swapped. This change maps `RETAIN` to the branch reference age and the snapshot retention time to the snapshot age, matching the Iceberg Spark implementation. Unit tests cover the option conversion, DDL routing, and the resulting Iceberg snapshot reference metadata. ### Release note Fix Iceberg branch metadata so `RETAIN` sets `max-ref-age-ms` and the snapshot retention time sets `max-snapshot-age-ms`. ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [x] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent 87156eb commit 2e7f79e

6 files changed

Lines changed: 25 additions & 18 deletions

File tree

‎fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/BranchOptions.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,11 @@ public class BranchOptions {
3232
Optional.empty());
3333

3434
private final Optional<Long> snapshotId;
35-
// retain time in milliseconds
35+
// RETAIN time in milliseconds, for the branch reference's maximum age
3636
private final Optional<Long> retain;
37+
// WITH SNAPSHOT RETENTION <n> SNAPSHOTS
3738
private final Optional<Integer> numSnapshots;
39+
// WITH SNAPSHOT RETENTION <time> in milliseconds, for the maximum snapshot age
3840
private final Optional<Long> retention;
3941

4042
public BranchOptions(Optional<Long> snapshotId,

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsDdlTest.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -304,13 +304,14 @@ public void testCreateBranchNullSnapshotUsesCurrent() {
304304
@Test
305305
public void testCreateBranchAppliesRetentionOptions() {
306306
long snap = createTableWithSnapshots("t1", 1);
307+
long maxRefAgeMs = 86400000L;
308+
long maxSnapshotAgeMs = 172800000L;
307309
ops.createOrReplaceBranch("db1", "t1",
308-
new BranchChange("b1", true, false, false, snap, 86400000L, 5, 172800000L));
310+
new BranchChange("b1", true, false, false, snap, maxSnapshotAgeMs, 5, maxRefAgeMs));
309311
SnapshotRef r = ref("t1", "b1");
310-
// retain -> maxSnapshotAgeMs, numSnapshots -> minSnapshotsToKeep, retention -> maxRefAgeMs (legacy mapping).
311-
Assertions.assertEquals(86400000L, r.maxSnapshotAgeMs());
312+
Assertions.assertEquals(maxSnapshotAgeMs, r.maxSnapshotAgeMs());
312313
Assertions.assertEquals(5, r.minSnapshotsToKeep());
313-
Assertions.assertEquals(172800000L, r.maxRefAgeMs());
314+
Assertions.assertEquals(maxRefAgeMs, r.maxRefAgeMs());
314315
}
315316

316317
@Test

‎fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ddl/BranchChange.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@
2424
* <p>The retention fields are nullable ({@code null} = not specified in the SQL, so the connector leaves the
2525
* corresponding setting untouched), mirroring the {@code Optional<>} fields of the source {@code BranchOptions}.
2626
* They are named after the snapshot-management knobs they drive, so a connector applies them 1:1:
27-
* {@code maxSnapshotAgeMs} (SQL {@code RETAIN}), {@code minSnapshotsToKeep} (SQL {@code WITH SNAPSHOT RETENTION
28-
* n SNAPSHOTS}), {@code maxRefAgeMs} (SQL {@code WITH SNAPSHOT RETENTION ... MINUTES}). A {@code null}
27+
* {@code maxRefAgeMs} (SQL {@code RETAIN}), {@code minSnapshotsToKeep} (SQL {@code WITH SNAPSHOT RETENTION
28+
* n SNAPSHOTS}), {@code maxSnapshotAgeMs} (SQL {@code WITH SNAPSHOT RETENTION ... MINUTES}). A {@code null}
2929
* {@code snapshotId} means "use the table's current snapshot", resolved by the connector against the live table.</p>
3030
*/
3131
public final class BranchChange {
@@ -72,7 +72,7 @@ public Long getSnapshotId() {
7272
return snapshotId;
7373
}
7474

75-
/** SQL {@code RETAIN} in ms, or {@code null} when unset. */
75+
/** SQL {@code WITH SNAPSHOT RETENTION ... MINUTES} in ms, or {@code null} when unset. */
7676
public Long getMaxSnapshotAgeMs() {
7777
return maxSnapshotAgeMs;
7878
}
@@ -82,7 +82,7 @@ public Integer getMinSnapshotsToKeep() {
8282
return minSnapshotsToKeep;
8383
}
8484

85-
/** SQL {@code WITH SNAPSHOT RETENTION ... MINUTES} in ms, or {@code null} when unset. */
85+
/** SQL {@code RETAIN} in ms, or {@code null} when unset. */
8686
public Long getMaxRefAgeMs() {
8787
return maxRefAgeMs;
8888
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorBranchTagConverter.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,11 @@ public final class ConnectorBranchTagConverter {
4040
private ConnectorBranchTagConverter() {
4141
}
4242

43-
/** Maps {@code BranchOptions.retain/numSnapshots/retention} to the snapshot-management knobs they drive. */
43+
/**
44+
* Maps {@code BranchOptions} to the snapshot-management knobs they drive. SQL
45+
* {@code RETAIN} controls the branch reference age, while the time after
46+
* {@code WITH SNAPSHOT RETENTION} controls the snapshot age.
47+
*/
4448
public static BranchChange toBranchChange(CreateOrReplaceBranchInfo info) {
4549
BranchOptions options = info.getBranchOptions();
4650
return new BranchChange(
@@ -49,9 +53,9 @@ public static BranchChange toBranchChange(CreateOrReplaceBranchInfo info) {
4953
info.getReplace(),
5054
info.getIfNotExists(),
5155
options.getSnapshotId().orElse(null),
52-
options.getRetain().orElse(null),
56+
options.getRetention().orElse(null), // WITH SNAPSHOT RETENTION <time>
5357
options.getNumSnapshots().orElse(null),
54-
options.getRetention().orElse(null));
58+
options.getRetain().orElse(null)); // RETAIN <time>
5559
}
5660

5761
public static TagChange toTagChange(CreateOrReplaceTagInfo info) {

‎fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorBranchTagConverterTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434

3535
/**
3636
* Unit tests for {@link ConnectorBranchTagConverter}: every nereids info/option field must land on the right
37-
* neutral carrier field (Rule 9), incl. the legacy {@code BranchOptions} naming (retain -> maxSnapshotAge,
38-
* numSnapshots -> minSnapshotsToKeep, retention -> maxRefAge), and absent options must become {@code null}.
37+
* neutral carrier field (Rule 9), incl. the legacy {@code BranchOptions} naming (retain -> maxRefAge,
38+
* numSnapshots -> minSnapshotsToKeep, retention -> maxSnapshotAge), and absent options must become {@code null}.
3939
*/
4040
public class ConnectorBranchTagConverterTest {
4141

@@ -50,9 +50,9 @@ public void testBranchFullOptions() {
5050
Assertions.assertFalse(b.isReplace());
5151
Assertions.assertTrue(b.isIfNotExists());
5252
Assertions.assertEquals(42L, b.getSnapshotId().longValue());
53-
Assertions.assertEquals(86400000L, b.getMaxSnapshotAgeMs().longValue());
53+
Assertions.assertEquals(86400000L, b.getMaxRefAgeMs().longValue());
5454
Assertions.assertEquals(5, b.getMinSnapshotsToKeep().intValue());
55-
Assertions.assertEquals(172800000L, b.getMaxRefAgeMs().longValue());
55+
Assertions.assertEquals(172800000L, b.getMaxSnapshotAgeMs().longValue());
5656
}
5757

5858
@Test

‎fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1339,9 +1339,9 @@ public void testCreateOrReplaceBranchRoutesConvertsAndRefreshes() throws Excepti
13391339
Assertions.assertFalse(b.isReplace());
13401340
Assertions.assertTrue(b.isIfNotExists());
13411341
Assertions.assertEquals(42L, b.getSnapshotId().longValue());
1342-
Assertions.assertEquals(86400000L, b.getMaxSnapshotAgeMs().longValue());
1342+
Assertions.assertEquals(86400000L, b.getMaxRefAgeMs().longValue());
13431343
Assertions.assertEquals(5, b.getMinSnapshotsToKeep().intValue());
1344-
Assertions.assertEquals(172800000L, b.getMaxRefAgeMs().longValue());
1344+
Assertions.assertEquals(172800000L, b.getMaxSnapshotAgeMs().longValue());
13451345
ArgumentCaptor<ExternalObjectLog> logCap = ArgumentCaptor.forClass(ExternalObjectLog.class);
13461346
Mockito.verify(mockEditLog).logRefreshExternalTable(logCap.capture());
13471347
Assertions.assertEquals("db1", logCap.getValue().getDbName());

0 commit comments

Comments
 (0)