Skip to content

Commit 65b5ad2

Browse files
committed
fix: keep ABFS URL passwords out of store keys, resolution errors and driver messages
object_store_authority kept an ABFS URL's password, so it reached the store cache key, the registration URL, debug lines and the deregister_store error. object_store_backend parsed the raw URL and normalize_object_store_url echoed input it could not parse. On the driver, AbfsAuthResolver put the URI into forwarded errors and a warning, and the mixed-store decline reason and objectStoreKey kept the raw authority. All of these now name a URL by scheme, container and host only. The URL helpers move to objectstore/url_redaction.rs, NativeConfig gains a matching displayAuthority, objectStoreKey matches the native key again, and the resolver scrubs the URI's password and query from Hadoop messages it forwards. Credentials only come from configuration, so URLs that differ only by password share a store.
1 parent c447c1e commit 65b5ad2

9 files changed

Lines changed: 434 additions & 80 deletions

File tree

‎native/core/src/parquet/objectstore/azure.rs‎

Lines changed: 3 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,16 @@
7171
7272
use log::debug;
7373
use std::collections::HashMap;
74-
use url::{Position, Url};
74+
use url::Url;
7575

7676
use object_store::{
7777
azure::{AzureConfigKey, MicrosoftAzureBuilder},
7878
path::Path,
7979
ObjectStore, ObjectStoreScheme,
8080
};
8181

82+
use super::url_redaction::{display_authority, redacted, without_credentials, without_secrets};
83+
8284
const MARKER_RESOLUTION: &str = "comet.azure.resolution";
8385
const MARKER_AUTH_TYPE: &str = "comet.azure.auth.type";
8486
const MARKER_OAUTH_PROVIDER_CLASS: &str = "comet.azure.oauth.provider.class";
@@ -328,50 +330,6 @@ fn authority_host(authority: &str) -> String {
328330
authority.trim_end_matches('/').to_string()
329331
}
330332

331-
// A URL's password and query (where a SAS token sits) can carry a credential, so no message
332-
// or URL handed to object_store, which quotes URLs in its errors, keeps them. The path is not
333-
// treated as secret.
334-
335-
/// `url` without its password, query and fragment. The username stays: in an ABFS URL it is the
336-
/// container, which `MicrosoftAzureBuilder` reads.
337-
fn without_secrets(url: &Url) -> Url {
338-
let mut stripped = url.clone();
339-
stripped.set_query(None);
340-
stripped.set_fragment(None);
341-
// Fails only for a URL that cannot have userinfo, so there is nothing to clear.
342-
let _ = stripped.set_password(None);
343-
stripped
344-
}
345-
346-
/// [`without_secrets`] without the username too, for a URL that may not be Azure, where the
347-
/// username can be an access key ID. `ObjectStoreScheme::parse` reads only the scheme, host and
348-
/// path.
349-
fn without_credentials(url: &Url) -> Url {
350-
let mut stripped = without_secrets(url);
351-
// Fails only for a URL that cannot have userinfo, so there is nothing to clear.
352-
let _ = stripped.set_username("");
353-
stripped
354-
}
355-
356-
/// `scheme://[container@]host[:port]` of an ABFS URL, for messages.
357-
fn display_authority(url: &Url) -> String {
358-
let safe = without_secrets(url);
359-
format!(
360-
"{}://{}",
361-
safe.scheme(),
362-
&safe[Position::BeforeUsername..Position::AfterPort]
363-
)
364-
}
365-
366-
/// `scheme://host[:port]` of a URL that may not be Azure, for messages.
367-
fn redacted(url: &Url) -> String {
368-
format!(
369-
"{}://{}",
370-
url.scheme(),
371-
&url[Position::BeforeHost..Position::AfterPort]
372-
)
373-
}
374-
375333
fn config_error(message: String) -> object_store::Error {
376334
object_store::Error::Generic {
377335
store: "MicrosoftAzure",

‎native/core/src/parquet/objectstore/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,4 @@ pub(crate) mod http_metrics;
2020
pub mod location_scoped;
2121
pub mod s3;
2222
pub mod s3_blob_fs_support;
23+
pub(crate) mod url_redaction;

‎native/core/src/parquet/objectstore/s3_blob_fs_support.rs‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,15 @@ pub(crate) fn normalize_object_store_url(
7777
url_str: &str,
7878
object_store_configs: &HashMap<String, String>,
7979
) -> Result<NormalizedObjectStoreUrl, ExecutionError> {
80-
let url = Url::parse(url_str)
81-
.map_err(|e| ExecutionError::GeneralError(format!("Error parsing URL {url_str}: {e}")))?;
80+
let url = Url::parse(url_str).map_err(|e| {
81+
// `url_str` may carry credentials, so only a valid scheme is named. `ParseError` names
82+
// the problem without quoting the input.
83+
let message = match scheme_of_unparsed(url_str) {
84+
Some(scheme) => format!("Error parsing {scheme} URL: {e}"),
85+
None => format!("Error parsing URL: {e}"),
86+
};
87+
ExecutionError::GeneralError(message)
88+
})?;
8289
if is_hdfs_scheme(&url, object_store_configs) {
8390
return Ok(NormalizedObjectStoreUrl { url, is_hdfs: true });
8491
}
@@ -95,6 +102,15 @@ pub(crate) fn normalize_object_store_url(
95102
})
96103
}
97104

105+
/// The text before `://` in a URL that failed to parse, if it is a valid scheme.
106+
fn scheme_of_unparsed(url_str: &str) -> Option<&str> {
107+
let (scheme, _) = url_str.split_once("://")?;
108+
let mut chars = scheme.chars();
109+
let is_scheme = chars.next().is_some_and(|c| c.is_ascii_alphabetic())
110+
&& chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'));
111+
is_scheme.then_some(scheme)
112+
}
113+
98114
/// True if `scheme` is a configured s3-compliant alias (`fs.comet.s3Compliant.schemes`; empty or
99115
/// unset means none). These route to `AmazonS3` but are not recognized by
100116
/// `ObjectStoreScheme::parse`. `s3a` is excluded -- object_store knows it and callers special-case
@@ -303,6 +319,29 @@ mod tests {
303319
configs
304320
}
305321

322+
#[test]
323+
fn parse_errors_name_only_a_valid_scheme() {
324+
let empty = HashMap::new();
325+
let error = |url: &str| match normalize_object_store_url(url, &empty) {
326+
Err(e) => e.to_string(),
327+
Ok(_) => panic!("{url} parsed"),
328+
};
329+
let err = error("s3a://alice:SECRET@bucket:notaport/x?sig=SAS-MARKER");
330+
assert!(
331+
err.contains("Error parsing s3a URL: invalid port number"),
332+
"{err}"
333+
);
334+
assert!(!err.contains("alice") && !err.contains("SECRET"), "{err}");
335+
// The text before `://` is not a scheme, so none of the input is echoed.
336+
let err = error("1alice:SECRET@host://x");
337+
assert!(err.contains("Error parsing URL: "), "{err}");
338+
assert!(!err.contains("alice") && !err.contains("SECRET"), "{err}");
339+
// Starts like a scheme, but '@' is not a scheme character.
340+
let err = error("alice@host://x");
341+
assert!(err.contains("Error parsing URL: "), "{err}");
342+
assert!(!err.contains("alice") && !err.contains("host"), "{err}");
343+
}
344+
306345
#[test]
307346
fn test_normalize_object_store_url() {
308347
let blob = configs_with("blob");
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! URL forms safe to put in messages, store keys and URLs handed to object_store, which quotes
19+
//! URLs in its errors. A URL's password and query (where a SAS token sits) can carry a
20+
//! credential. The path is not treated as secret.
21+
22+
use std::borrow::Cow;
23+
use url::{Position, Url};
24+
25+
/// `url` without its password, query and fragment. The username stays: in an ABFS URL it is the
26+
/// container, which `MicrosoftAzureBuilder` reads.
27+
pub(crate) fn without_secrets(url: &Url) -> Url {
28+
let mut stripped = url.clone();
29+
stripped.set_query(None);
30+
stripped.set_fragment(None);
31+
// Fails only for a URL that cannot have userinfo, so there is nothing to clear.
32+
let _ = stripped.set_password(None);
33+
stripped
34+
}
35+
36+
/// [`without_secrets`] without the username too, for a URL that may not be Azure, where the
37+
/// username can be an access key ID. `ObjectStoreScheme::parse` reads only the scheme, host and
38+
/// path.
39+
pub(crate) fn without_credentials(url: &Url) -> Url {
40+
let mut stripped = without_secrets(url);
41+
// Fails only for a URL that cannot have userinfo, so there is nothing to clear.
42+
let _ = stripped.set_username("");
43+
stripped
44+
}
45+
46+
/// `[username@]host[:port]` of `url`, without the password.
47+
pub(crate) fn authority_without_password(url: &Url) -> Cow<'_, str> {
48+
if url.password().is_none() {
49+
return Cow::Borrowed(&url[Position::BeforeUsername..Position::AfterPort]);
50+
}
51+
Cow::Owned(without_secrets(url)[Position::BeforeUsername..Position::AfterPort].to_string())
52+
}
53+
54+
/// `scheme://[container@]host[:port]` of an ABFS URL, for messages.
55+
pub(crate) fn display_authority(url: &Url) -> String {
56+
format!("{}://{}", url.scheme(), authority_without_password(url))
57+
}
58+
59+
/// `scheme://host[:port]` of a URL that may not be Azure, for messages.
60+
pub(crate) fn redacted(url: &Url) -> String {
61+
format!(
62+
"{}://{}",
63+
url.scheme(),
64+
&url[Position::BeforeHost..Position::AfterPort]
65+
)
66+
}

0 commit comments

Comments
 (0)