Skip to content

Commit e850bff

Browse files
committed
[fix](lance) Carry the endpoint under both keys object_store consults
Review of the previous commit turned up a hole in it. `aws_endpoint_url_s3` is not an alias of the generic endpoint: object_store parses it into a config key of its own and always prefers it, // object_store-0.13.2 src/aws/builder.rs let endpoint = self.s3_endpoint.or(self.endpoint); so folding it onto the generic entry, as that commit did, replaced a precedence rule with map order. A namespace vending the standard AWS pair - a global `AWS_ENDPOINT_URL` plus an `AWS_ENDPOINT_URL_S3` override - would land on either value depending on iteration, and `allow_http` would follow whichever won. Worse, the map never contained that spelling at all, so `with_env_s3` was free to fill it from the environment, and object_store would then prefer it over the catalog's endpoint. A stray `AWS_ENDPOINT_URL_S3` in the FE or BE process silently redirected every request - the exact failure the previous commit set out to close, on the one key its approach could not reach. Both go away by writing the resolved endpoint under both spellings: object_store reads the same value whichever key it consults, `with_env_s3` finds both occupied, and OpenDAL ignores the S3-specific one as an unknown field. A namespace that vends both spellings now resolves to the S3-specific value, which is the one object_store itself would have used. Also from the review: - The NUL and protected-key log lines now name the offending key, with the NULs rendered, since an unnamed dropped credential surfaces later as an opaque 403. - allow_http gets a note that its canonical spelling buys no protection from the environment: `StorageOptions::new` overwrites that key outright from AWS_ALLOW_HTTP before `with_env_s3` runs. - testAllowHttpIsRetractedWhenTheEndpointBecomesHttps now covers the canonical spelling too. It only exercised `endpoint`, so the alias table's identity rows were uncovered: dropping `aws_endpoint -> aws_endpoint` left a stale allow_http=true behind a vended HTTPS endpoint and broke no test. Claude-Session: https://claude.ai/code/session_01PfVSEUJsfWwKy7WEuBaFCG
1 parent ed17099 commit e850bff

2 files changed

Lines changed: 115 additions & 36 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceStorageOptions.java‎

Lines changed: 58 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,21 @@ public final class LanceStorageOptions {
4747
private static final Logger LOG = LogManager.getLogger(LanceStorageOptions.class);
4848

4949
private static final String ENDPOINT = "aws_endpoint";
50+
/**
51+
* The S3-specific endpoint, which object_store keeps as a config key of its own and prefers
52+
* over the generic one: {@code let endpoint = self.s3_endpoint.or(self.endpoint)}. It carries
53+
* the same value as {@link #ENDPOINT} rather than a meaning of its own here.
54+
*/
55+
private static final String S3_ENDPOINT = "aws_endpoint_url_s3";
5056
private static final String VIRTUAL_HOSTED_STYLE = "aws_virtual_hosted_style_request";
51-
/** Not prefixed: object_store reports this shared client option as canonical under this name. */
57+
/**
58+
* Not prefixed: object_store reports this shared client option as canonical under this name.
59+
*
60+
* <p>Unlike the options below, the spelling buys no protection from the environment here.
61+
* {@code StorageOptions::new} overwrites this key outright from {@code AWS_ALLOW_HTTP} or
62+
* {@code AZURE_STORAGE_ALLOW_HTTP} before {@code with_env_s3} ever runs, so whatever is derived
63+
* for it can be overridden per process.
64+
*/
5265
private static final String ALLOW_HTTP = "allow_http";
5366

5467
/**
@@ -106,10 +119,10 @@ public final class LanceStorageOptions {
106119
.put("endpoint_url", ENDPOINT)
107120
.put("aws_endpoint", ENDPOINT)
108121
.put("aws_endpoint_url", ENDPOINT)
109-
// object_store parses this one into a config key of its own that wins over the generic
110-
// endpoint: `let endpoint = self.s3_endpoint.or(self.endpoint)`. Left alone it would not
111-
// displace the catalog's endpoint, and allow_http would be derived from the losing one.
112-
.put("aws_endpoint_url_s3", ENDPOINT)
122+
// Resolved onto the same entry, then written back under both spellings by
123+
// withResolvedEndpoint. Left alone it would not displace the catalog's endpoint, and
124+
// allow_http would be derived from the entry object_store was about to discard.
125+
.put(S3_ENDPOINT, ENDPOINT)
113126
.put("region", "aws_region")
114127
.put("aws_region", "aws_region")
115128
.put("virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE)
@@ -150,7 +163,7 @@ public static Map<String, String> toLanceOptions(Map<String, String> backendProp
150163
if (usePathStyle != null && !usePathStyle.isEmpty()) {
151164
result.put(VIRTUAL_HOSTED_STYLE, String.valueOf(!Boolean.parseBoolean(usePathStyle)));
152165
}
153-
return withDerivedAllowHttp(result);
166+
return withResolvedEndpoint(result);
154167
}
155168

156169
/**
@@ -168,30 +181,42 @@ public static Map<String, String> mergeVended(Map<String, String> lanceOptions,
168181

169182
Map<String, String> accepted = new HashMap<>();
170183
Set<String> superseded = new HashSet<>();
171-
vendedOptions.forEach((key, value) -> {
184+
String s3SpecificEndpoint = null;
185+
for (Map.Entry<String, String> vended : vendedOptions.entrySet()) {
186+
String key = vended.getKey();
187+
String value = vended.getValue();
172188
if (key == null || value == null || value.isEmpty()) {
173-
return;
189+
continue;
174190
}
175191
if (key.indexOf('\0') >= 0 || value.indexOf('\0') >= 0) {
176192
// lance-c reads these as C strings, so a NUL would truncate the option there while
177193
// leaving it unrecognized here - the two halves would disagree about the key.
178-
LOG.warn("Ignoring Lance storage option vended by the namespace because its key or "
179-
+ "value contains a NUL character");
180-
return;
194+
LOG.warn("Ignoring Lance storage option '{}' vended by the namespace because its "
195+
+ "key or value contains a NUL character", withoutNul(key));
196+
continue;
181197
}
182198
String lowerCased = key.toLowerCase(Locale.ROOT);
183199
if (PROTECTED_KEYS.contains(lowerCased)) {
184200
LOG.warn("Ignoring Lance storage option '{}' vended by the namespace because it "
185201
+ "would change which data is read", key);
186-
return;
202+
continue;
203+
}
204+
if (S3_ENDPOINT.equals(lowerCased)) {
205+
s3SpecificEndpoint = value;
187206
}
188207
String canonical = CANONICAL_BY_ALIAS.get(lowerCased);
189208
if (canonical != null) {
190209
superseded.add(canonical);
191210
}
192211
accepted.put(canonical != null && !AMBIGUOUS_ALIASES.contains(lowerCased)
193212
? canonical : key, value);
194-
});
213+
}
214+
// A namespace vending both endpoint spellings means the S3-specific one, because that is
215+
// the one object_store would have used. Resolving it by map order instead would leave the
216+
// FE and the BE free to pick differently.
217+
if (s3SpecificEndpoint != null) {
218+
accepted.put(ENDPOINT, s3SpecificEndpoint);
219+
}
195220

196221
// Drop the catalog's spelling of every option the namespace just supplied, so the two can
197222
// never reach Lance as competing entries for one config key.
@@ -202,23 +227,39 @@ public static Map<String, String> mergeVended(Map<String, String> lanceOptions,
202227
result.remove(ALLOW_HTTP);
203228
}
204229
result.putAll(accepted);
205-
return withDerivedAllowHttp(result);
230+
return withResolvedEndpoint(result);
206231
}
207232

208233
/**
209-
* Allows plain HTTP when the endpoint in use asks for it.
234+
* Writes the endpoint under both spellings object_store keeps a config key for, and allows
235+
* plain HTTP when that endpoint asks for it.
210236
*
211237
* <p>Applied after merging, because a namespace can replace the endpoint the catalog was
212238
* configured with - or supply the only one there is.
239+
*
240+
* <p>Emitting only the generic spelling would leave the S3-specific one for
241+
* {@code with_env_s3} to fill from {@code AWS_ENDPOINT_URL_S3}, and object_store prefers that
242+
* key, so a stray environment variable would silently redirect every request. Both carry the
243+
* same value, so which one object_store picks stops mattering.
213244
*/
214-
private static Map<String, String> withDerivedAllowHttp(Map<String, String> options) {
245+
private static Map<String, String> withResolvedEndpoint(Map<String, String> options) {
215246
String endpoint = options.get(ENDPOINT);
216-
if (endpoint != null && endpoint.startsWith("http://")) {
247+
if (endpoint == null) {
248+
options.remove(S3_ENDPOINT);
249+
return options;
250+
}
251+
options.put(S3_ENDPOINT, endpoint);
252+
if (endpoint.startsWith("http://")) {
217253
options.putIfAbsent(ALLOW_HTTP, "true");
218254
}
219255
return options;
220256
}
221257

258+
/** Renders a key that carries a NUL for a log line, since the NUL itself is what is wrong. */
259+
private static String withoutNul(String key) {
260+
return key.replace('\0', '?');
261+
}
262+
222263
private static void putIfNotEmpty(Map<String, String> target, String key, String value) {
223264
if (value != null && !value.isEmpty()) {
224265
target.put(key, value);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceStorageOptionsTest.java‎

Lines changed: 57 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@
2323
import java.util.Arrays;
2424
import java.util.Collections;
2525
import java.util.HashMap;
26+
import java.util.LinkedHashMap;
2627
import java.util.Locale;
2728
import java.util.Map;
2829
import java.util.TreeSet;
30+
import java.util.stream.Collectors;
2931

3032
public class LanceStorageOptionsTest {
3133

@@ -49,6 +51,8 @@ public void testMinioStorageOptionMapping() {
4951
Assertions.assertEquals("sk", options.get("aws_secret_access_key"));
5052
Assertions.assertEquals("token", options.get("aws_session_token"));
5153
Assertions.assertEquals("http://minio:9000", options.get("aws_endpoint"));
54+
// object_store prefers its S3-specific endpoint key, so both carry the same value.
55+
Assertions.assertEquals("http://minio:9000", options.get("aws_endpoint_url_s3"));
5256
Assertions.assertEquals("us-east-1", options.get("aws_region"));
5357
Assertions.assertEquals("true", options.get("allow_http"));
5458
Assertions.assertEquals("false", options.get("aws_virtual_hosted_style_request"));
@@ -71,7 +75,7 @@ public void testEmittedOptionsUseTheSpellingThatSuppressesTheEnvironment() {
7175
// because object_store carries it as a shared client option rather than an S3 one.
7276
Assertions.assertEquals(
7377
new TreeSet<>(Arrays.asList("aws_access_key_id", "aws_secret_access_key",
74-
"aws_session_token", "aws_endpoint", "aws_region",
78+
"aws_session_token", "aws_endpoint", "aws_endpoint_url_s3", "aws_region",
7579
"aws_virtual_hosted_style_request", "allow_http")),
7680
new TreeSet<>(options.keySet()));
7781
}
@@ -141,13 +145,10 @@ public void testVendedPrefixedOptionsReplaceTheCatalogEntry() {
141145
/**
142146
* object_store accepts five spellings of the endpoint and four of the session token. Any one
143147
* this class fails to recognize reintroduces the race, so the whole equivalence class has to
144-
* collapse onto a single entry.
148+
* collapse - onto one value, carried by the two keys object_store actually consults.
145149
*/
146150
@Test
147-
public void testEveryAcceptedAliasCollapsesOntoOneEntry() {
148-
// aws_endpoint_url_s3 parses into a config key of its own that object_store prefers over
149-
// the generic endpoint, so leaving it beside one would not even be a coin toss - the vended
150-
// value would win while allow_http was derived from the catalog's.
151+
public void testEveryAcceptedAliasCollapsesOntoOneValue() {
151152
for (String alias : new String[] {"endpoint", "endpoint_url", "aws_endpoint",
152153
"aws_endpoint_url", "aws_endpoint_url_s3", "ENDPOINT", "AWS_Endpoint_Url"}) {
153154
Map<String, String> vended = new HashMap<>();
@@ -156,12 +157,41 @@ public void testEveryAcceptedAliasCollapsesOntoOneEntry() {
156157
Map<String, String> merged = LanceStorageOptions.mergeVended(
157158
LanceStorageOptions.toLanceOptions(minioCatalogProperties()), vended);
158159

159-
long endpoints = merged.entrySet().stream()
160-
.filter(e -> e.getKey().toLowerCase(Locale.ROOT).contains("endpoint"))
161-
.count();
162-
Assertions.assertEquals(1, endpoints, "alias " + alias + " left a competing entry");
160+
Assertions.assertEquals(
161+
new TreeSet<>(Arrays.asList("aws_endpoint", "aws_endpoint_url_s3")),
162+
merged.keySet().stream()
163+
.filter(key -> key.toLowerCase(Locale.ROOT).contains("endpoint"))
164+
.collect(Collectors.toCollection(TreeSet::new)),
165+
"alias " + alias + " left a competing entry");
163166
Assertions.assertEquals("http://127.0.0.1:9000", merged.get("aws_endpoint"),
164167
"alias " + alias + " did not win");
168+
Assertions.assertEquals("http://127.0.0.1:9000", merged.get("aws_endpoint_url_s3"),
169+
"alias " + alias + " did not reach the key object_store prefers");
170+
}
171+
}
172+
173+
/**
174+
* object_store resolves the endpoint as {@code s3_endpoint.or(endpoint)}, so a namespace that
175+
* vends both spellings means the S3-specific one. Both have to end up on the same value anyway,
176+
* or the FE and the BE could each read a different key and disagree.
177+
*/
178+
@Test
179+
public void testS3SpecificEndpointWinsOverAGenericOneVendedBesideIt() {
180+
for (boolean s3First : new boolean[] {true, false}) {
181+
Map<String, String> vended = new LinkedHashMap<>();
182+
if (s3First) {
183+
vended.put("aws_endpoint_url_s3", "http://minio:9000");
184+
vended.put("endpoint", "https://generic.example.com");
185+
} else {
186+
vended.put("endpoint", "https://generic.example.com");
187+
vended.put("aws_endpoint_url_s3", "http://minio:9000");
188+
}
189+
190+
Map<String, String> merged = LanceStorageOptions.mergeVended(
191+
LanceStorageOptions.toLanceOptions(minioCatalogProperties()), vended);
192+
Assertions.assertEquals("http://minio:9000", merged.get("aws_endpoint"));
193+
Assertions.assertEquals("http://minio:9000", merged.get("aws_endpoint_url_s3"));
194+
Assertions.assertEquals("true", merged.get("allow_http"));
165195
}
166196
}
167197

@@ -238,16 +268,24 @@ public void testAllowHttpFollowsTheEndpointActuallyUsed() {
238268
*/
239269
@Test
240270
public void testAllowHttpIsRetractedWhenTheEndpointBecomesHttps() {
241-
Map<String, String> catalogOptions =
242-
LanceStorageOptions.toLanceOptions(minioCatalogProperties());
243-
Assertions.assertEquals("true", catalogOptions.get("allow_http"));
271+
// Every spelling has to retract it, including the canonical one the catalog itself emits:
272+
// an alias that resolves to no canonical name would leave the derived value standing.
273+
for (String alias : new String[] {"endpoint", "aws_endpoint", "aws_endpoint_url_s3"}) {
274+
Map<String, String> catalogOptions =
275+
LanceStorageOptions.toLanceOptions(minioCatalogProperties());
276+
Assertions.assertEquals("true", catalogOptions.get("allow_http"));
244277

245-
Map<String, String> vended = new HashMap<>();
246-
vended.put("endpoint", "https://s3.amazonaws.com");
247-
248-
Map<String, String> merged = LanceStorageOptions.mergeVended(catalogOptions, vended);
249-
Assertions.assertEquals("https://s3.amazonaws.com", merged.get("aws_endpoint"));
250-
Assertions.assertNull(merged.get("allow_http"));
278+
Map<String, String> vended = new HashMap<>();
279+
vended.put(alias, "https://s3.amazonaws.com");
280+
281+
Map<String, String> merged = LanceStorageOptions.mergeVended(catalogOptions, vended);
282+
Assertions.assertEquals("https://s3.amazonaws.com", merged.get("aws_endpoint"),
283+
"alias " + alias + " did not replace the endpoint");
284+
Assertions.assertEquals("https://s3.amazonaws.com", merged.get("aws_endpoint_url_s3"),
285+
"alias " + alias + " left a stale S3-specific endpoint");
286+
Assertions.assertNull(merged.get("allow_http"),
287+
"alias " + alias + " left allow_http standing");
288+
}
251289
}
252290

253291
/**

0 commit comments

Comments
 (0)