Skip to content

Commit 4728d69

Browse files
morningmanclaude
andcommitted
[fix](fe) Keep the Iceberg SDK manifest cache off under a metadata cache weight limit
### What problem does this PR solve? Issue Number: None Related PR: #66717, #67726 Problem Summary: Under a Doris metadata cache weight limit the Iceberg catalog left `io.manifest.cache-enabled` out unless the user set it, so the SDK manifest content cache, which keeps up to `io.manifest.cache.max-total-bytes` of manifests per FileIO outside the Doris limit, stayed at its default of off. An Iceberg REST catalog, however, merges the server's `/config` response into the client properties, and its defaults fill in any key the client leaves out: a server default of `true` turned the cache on. The catalog now writes `false` explicitly, which the merge keeps over the server's defaults. The server's overrides and a table's own config still take precedence over client properties, as the REST protocol intends, and an explicit user setting still wins as before. ### Release note Under an external metadata cache weight limit, an Iceberg catalog keeps the SDK manifest cache off even when an Iceberg REST server's /config defaults enable it, unless the catalog sets io.manifest.cache-enabled itself. ### Check List (For Author) - Test: Unit Test - IcebergCatalogFactoryTest: under a weight limit the catalog properties carry io.manifest.cache-enabled=false, and merging them with a REST /config response whose defaults enable the cache keeps it off. Leaving the key out makes it fail. - Behavior changed: Yes (see Release note) - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent b6db733 commit 4728d69

2 files changed

Lines changed: 33 additions & 15 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -163,8 +163,10 @@ public static Map<String, String> buildBaseCatalogProperties(Map<String, String>
163163
* {@code io.manifest.cache-enabled} directly — derive it to {@code "true"} from the FE meta-cache
164164
* spec ({@code meta.cache.iceberg.manifest.*}) using the same {@code enable && ttl != 0 &&
165165
* capacity != 0} rule. Default-disabled (legacy {@code DEFAULT_ICEBERG_MANIFEST_CACHE_ENABLE}).
166-
* Nothing is derived under a metadata cache weight limit: the SDK cache keeps up to
167-
* {@code io.manifest.cache.max-total-bytes} of manifest content per FileIO outside that limit.
166+
* Under a metadata cache weight limit the SDK cache, which keeps up to
167+
* {@code io.manifest.cache.max-total-bytes} of manifest content per FileIO outside that limit, is not
168+
* derived but switched off explicitly: an Iceberg REST server's {@code /config} defaults fill in a key the
169+
* client leaves out. The server's overrides and a table's own config still take precedence.
168170
*/
169171
private static void appendManifestCacheProperties(Map<String, String> props, Map<String, String> opts,
170172
boolean metaCacheWeightLimited) {
@@ -173,14 +175,19 @@ private static void appendManifestCacheProperties(Map<String, String> props, Map
173175
copyIfPresent(props, opts, CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS);
174176
copyIfPresent(props, opts, CatalogProperties.IO_MANIFEST_CACHE_MAX_TOTAL_BYTES);
175177
copyIfPresent(props, opts, CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH);
176-
if (!hasExplicitEnabled && !metaCacheWeightLimited) {
177-
CacheSpec spec = CacheSpec.fromProperties(props,
178-
IcebergConnector.MANIFEST_CACHE_ENABLE, DEFAULT_MANIFEST_CACHE_ENABLE,
179-
IcebergConnector.MANIFEST_CACHE_TTL_SECOND, DEFAULT_MANIFEST_CACHE_TTL_SECOND,
180-
IcebergConnector.MANIFEST_CACHE_CAPACITY, DEFAULT_MANIFEST_CACHE_CAPACITY);
181-
if (CacheSpec.isCacheEnabled(spec.isEnable(), spec.getTtlSecond(), spec.getCapacity())) {
182-
opts.put(CatalogProperties.IO_MANIFEST_CACHE_ENABLED, "true");
183-
}
178+
if (hasExplicitEnabled) {
179+
return;
180+
}
181+
if (metaCacheWeightLimited) {
182+
opts.put(CatalogProperties.IO_MANIFEST_CACHE_ENABLED, "false");
183+
return;
184+
}
185+
CacheSpec spec = CacheSpec.fromProperties(props,
186+
IcebergConnector.MANIFEST_CACHE_ENABLE, DEFAULT_MANIFEST_CACHE_ENABLE,
187+
IcebergConnector.MANIFEST_CACHE_TTL_SECOND, DEFAULT_MANIFEST_CACHE_TTL_SECOND,
188+
IcebergConnector.MANIFEST_CACHE_CAPACITY, DEFAULT_MANIFEST_CACHE_CAPACITY);
189+
if (CacheSpec.isCacheEnabled(spec.isEnable(), spec.getTtlSecond(), spec.getCapacity())) {
190+
opts.put(CatalogProperties.IO_MANIFEST_CACHE_ENABLED, "true");
184191
}
185192
}
186193

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.hadoop.conf.Configuration;
2828
import org.apache.hadoop.hive.conf.HiveConf;
2929
import org.apache.iceberg.aws.AwsClientProperties;
30+
import org.apache.iceberg.rest.responses.ConfigResponse;
3031
import org.junit.jupiter.api.Assertions;
3132
import org.junit.jupiter.api.Test;
3233
import org.junit.jupiter.api.io.TempDir;
@@ -226,18 +227,28 @@ public void buildBaseMetaCacheEnabledButZeroTtlDoesNotEnable() {
226227
}
227228

228229
@Test
229-
public void metaCacheWeightLimitSkipsManifestCacheDerivation() {
230+
public void metaCacheWeightLimitTurnsTheSdkManifestCacheOff() {
230231
// WHY: the SDK manifest content cache keeps up to io.manifest.cache.max-total-bytes of manifests per FileIO
231232
// outside every Doris metadata cache weight limit, so under a limit it is not derived from
232-
// meta.cache.iceberg.manifest.* (branch-4.x AbstractIcebergProperties). An explicit user value still wins.
233-
// MUTATION: ignoring the limit flag -> "true" is derived -> red.
233+
// meta.cache.iceberg.manifest.* (branch-4.x AbstractIcebergProperties) but switched off outright, because
234+
// an Iceberg REST server's /config defaults fill in a key the client leaves out. An explicit user value
235+
// still wins. MUTATION: ignoring the limit flag -> "true" is derived -> red; leaving the key out -> the
236+
// REST server's default turns the cache on -> red.
234237
Map<String, String> metaCacheOnly = props("iceberg.catalog.type", "hadoop", "warehouse", "s3://b/wh",
235238
"meta.cache.iceberg.manifest.enable", "true");
236-
Assertions.assertNull(IcebergCatalogFactory.buildBaseCatalogProperties(metaCacheOnly, true)
239+
Assertions.assertEquals("false", IcebergCatalogFactory.buildBaseCatalogProperties(metaCacheOnly, true)
237240
.get("io.manifest.cache-enabled"));
238-
Assertions.assertNull(IcebergCatalogFactory.buildCatalogProperties(
241+
Assertions.assertEquals("false", IcebergCatalogFactory.buildCatalogProperties(
239242
IcebergCatalogProperties.of(metaCacheOnly), Optional.empty(), true).get("io.manifest.cache-enabled"));
240243

244+
Map<String, String> rest = IcebergCatalogFactory.buildCatalogProperties(
245+
IcebergCatalogProperties.of(props("iceberg.catalog.type", "rest", "uri", "https://rest")),
246+
Optional.empty(), true);
247+
ConfigResponse serverDefaultsOn = ConfigResponse.builder()
248+
.withDefault("io.manifest.cache-enabled", "true").build();
249+
Assertions.assertEquals("false", serverDefaultsOn.merge(rest).get("io.manifest.cache-enabled"),
250+
"a REST server's /config default must not turn the SDK manifest cache back on");
251+
241252
Map<String, String> explicit = new HashMap<>(metaCacheOnly);
242253
explicit.put("io.manifest.cache-enabled", "true");
243254
Assertions.assertEquals("true", IcebergCatalogFactory.buildBaseCatalogProperties(explicit, true)

0 commit comments

Comments
 (0)