Skip to content

Commit dcc61a5

Browse files
committed
perf: key the native object store cache on the settings each store reads
The native scan now receives the object store settings of every scheme it reads. Hashing the whole map gave one bucket a different cache entry, and a second HTTP client and credential fetch, depending on what else the scan read. Each backend now hashes and receives only the keys it reads: fs.s3a.* for S3; fs.azure.* and the comet.azure.* resolution markers for Azure; and for every store the fs.comet.* scheme lists that decide routing.
1 parent 2b9c9c9 commit dcc61a5

1 file changed

Lines changed: 145 additions & 4 deletions

File tree

‎native/core/src/parquet/parquet_support.rs‎

Lines changed: 145 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -793,7 +793,7 @@ fn create_hdfs_object_store(
793793

794794
/// Cache identity: `(scheme://[container@]host:port, config_hash, hdfs_backend)`.
795795
/// Native `s3a` is normalized to `s3`; Hadoop-selected schemes keep their spelling.
796-
/// The hash covers the object-store configuration. The boolean is `true` for the
796+
/// The hash covers the options the backend reads. The boolean is `true` for the
797797
/// Hadoop backend (including custom schemes routed through Hadoop), `false` for native.
798798
type ObjectStoreCacheKey = (String, u64, bool);
799799
type ObjectStoreCache = RwLock<HashMap<ObjectStoreCacheKey, Arc<dyn ObjectStore>>>;
@@ -857,6 +857,23 @@ fn hash_object_store_configs(configs: &HashMap<String, String>) -> u64 {
857857
hasher.finish()
858858
}
859859

860+
/// The option prefixes the store built for `scheme` reads. A scan forwards the options of every
861+
/// scheme it reads, so the cache identity and the store use only these, and a bucket keeps one
862+
/// store whatever other schemes the scan reads. `fs.comet.*` selects the backend.
863+
fn backend_option_prefixes(scheme: &str, is_hdfs: bool) -> &'static [&'static str] {
864+
if is_hdfs {
865+
&["fs.comet."]
866+
} else if scheme == "s3" {
867+
&["fs.comet.", "fs.s3a."]
868+
} else if is_azure_scheme(scheme) {
869+
// `comet.azure.*` carries the driver's credential resolution.
870+
&["fs.comet.", "fs.azure.", "comet.azure."]
871+
} else {
872+
// `parse_url` reads no options.
873+
&["fs.comet."]
874+
}
875+
}
876+
860877
/// The selected backend, independent of the URL used to register it in DataFusion.
861878
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
862879
pub(crate) enum ObjectStoreBackend {
@@ -910,7 +927,13 @@ pub(crate) fn prepare_object_store_with_configs(
910927
let scheme = url.scheme();
911928
let url_key = object_store_url_key(&url);
912929

913-
let config_hash = hash_object_store_configs(object_store_configs);
930+
let prefixes = backend_option_prefixes(scheme, is_hdfs_scheme);
931+
let backend_configs: HashMap<String, String> = object_store_configs
932+
.iter()
933+
.filter(|(key, _)| prefixes.iter().any(|prefix| key.starts_with(prefix)))
934+
.map(|(key, value)| (key.clone(), value.clone()))
935+
.collect();
936+
let config_hash = hash_object_store_configs(&backend_configs);
914937
let cache_key = (url_key.clone(), config_hash, is_hdfs_scheme);
915938

916939
// Check the cache first to reuse existing object store instances.
@@ -933,9 +956,9 @@ pub(crate) fn prepare_object_store_with_configs(
933956
let (store, path): (Box<dyn ObjectStore>, Path) = if is_hdfs_scheme {
934957
create_hdfs_object_store(&url)
935958
} else if scheme == "s3" {
936-
objectstore::s3::create_store(&url, object_store_configs, Duration::from_secs(300))
959+
objectstore::s3::create_store(&url, &backend_configs, Duration::from_secs(300))
937960
} else if is_azure_scheme(scheme) {
938-
objectstore::azure::create_store(&url, object_store_configs)
961+
objectstore::azure::create_store(&url, &backend_configs)
939962
} else {
940963
parse_url(&url)
941964
}
@@ -1393,6 +1416,124 @@ mod tests {
13931416
object_store_cache().write().unwrap().remove(&key);
13941417
}
13951418

1419+
/// A scan forwards the options of every scheme it reads, so the S3 store of a bucket must not
1420+
/// depend on keys only other backends read, while a changed `fs.s3a.*` key selects another
1421+
/// store. Seeds and removes two in-memory cache entries; no remote requests are performed.
1422+
#[test]
1423+
fn s3_store_cache_identity_ignores_options_of_other_backends() {
1424+
let bucket = "comet-isolation-other-backend-options";
1425+
let s3_options = HashMap::from([
1426+
(
1427+
"fs.s3a.endpoint".to_string(),
1428+
"http://localhost:9000".to_string(),
1429+
),
1430+
("fs.comet.libhdfs.schemes".to_string(), "hdfs".to_string()),
1431+
]);
1432+
let mut changed_s3_options = s3_options.clone();
1433+
changed_s3_options.insert("fs.s3a.endpoint".into(), "http://localhost:9001".into());
1434+
let mut scan_options = s3_options.clone();
1435+
scan_options.insert("fs.gs.project.id".into(), "gcp-project".into());
1436+
scan_options.insert(
1437+
"fs.azure.account.key.acct.blob.core.windows.net".into(),
1438+
"azure-key".into(),
1439+
);
1440+
1441+
let seeded: Vec<(super::ObjectStoreCacheKey, Arc<dyn ObjectStore>)> =
1442+
[&s3_options, &changed_s3_options]
1443+
.iter()
1444+
.map(|options| {
1445+
let key = (
1446+
format!("s3://{bucket}"),
1447+
hash_object_store_configs(options),
1448+
false,
1449+
);
1450+
(key, Arc::new(InMemory::new()) as Arc<dyn ObjectStore>)
1451+
})
1452+
.collect();
1453+
for (key, store) in &seeded {
1454+
object_store_cache()
1455+
.write()
1456+
.unwrap()
1457+
.insert(key.clone(), Arc::clone(store));
1458+
}
1459+
1460+
let runtime = Arc::new(RuntimeEnv::default());
1461+
for (options, expected) in [
1462+
(&s3_options, &seeded[0].1),
1463+
(&scan_options, &seeded[0].1),
1464+
(&changed_s3_options, &seeded[1].1),
1465+
] {
1466+
let (url, _, _) = prepare_object_store_with_configs(
1467+
Arc::clone(&runtime),
1468+
format!("s3a://{bucket}/file.parquet"),
1469+
options,
1470+
)
1471+
.unwrap();
1472+
assert!(Arc::ptr_eq(&runtime.object_store(&url).unwrap(), expected));
1473+
}
1474+
for (key, _) in &seeded {
1475+
object_store_cache().write().unwrap().remove(key);
1476+
}
1477+
}
1478+
1479+
/// The Azure store of a container depends on the `fs.azure.*` and `comet.azure.*` options only.
1480+
/// Seeds and removes three in-memory cache entries; no remote requests are performed.
1481+
#[test]
1482+
fn azure_store_cache_identity_ignores_options_of_other_backends() {
1483+
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
1484+
1485+
let container = "comet-isolation-azure-options";
1486+
let azure_options = HashMap::from([("fs.azure.account.key".into(), "c2VjcmV0".into())]);
1487+
let mut changed_azure_options = azure_options.clone();
1488+
changed_azure_options.insert("fs.azure.account.key".into(), "b3RoZXI=".into());
1489+
let mut resolved_options = azure_options.clone();
1490+
resolved_options.insert("comet.azure.auth.type".into(), "SharedKey".into());
1491+
let mut scan_options = azure_options.clone();
1492+
scan_options.insert("fs.s3a.access.key".into(), "s3-access-key".into());
1493+
scan_options.insert("fs.gs.project.id".into(), "gcp-project".into());
1494+
1495+
let seeded: Vec<(super::ObjectStoreCacheKey, Arc<dyn ObjectStore>)> =
1496+
[&azure_options, &changed_azure_options, &resolved_options]
1497+
.iter()
1498+
.map(|options| {
1499+
let key = (
1500+
format!("abfss://{container}@account.dfs.core.windows.net"),
1501+
hash_object_store_configs(options),
1502+
false,
1503+
);
1504+
(key, Arc::new(InMemory::new()) as Arc<dyn ObjectStore>)
1505+
})
1506+
.collect();
1507+
for (key, store) in &seeded {
1508+
object_store_cache()
1509+
.write()
1510+
.unwrap()
1511+
.insert(key.clone(), Arc::clone(store));
1512+
}
1513+
1514+
let runtime = RuntimeEnvBuilder::new()
1515+
.with_object_store_registry(Arc::new(super::CometObjectStoreRegistry::default()))
1516+
.build_arc()
1517+
.unwrap();
1518+
for (options, expected) in [
1519+
(&azure_options, &seeded[0].1),
1520+
(&scan_options, &seeded[0].1),
1521+
(&changed_azure_options, &seeded[1].1),
1522+
(&resolved_options, &seeded[2].1),
1523+
] {
1524+
let (url, _, _) = prepare_object_store_with_configs(
1525+
Arc::clone(&runtime),
1526+
format!("abfss://{container}@account.dfs.core.windows.net/file.parquet"),
1527+
options,
1528+
)
1529+
.unwrap();
1530+
assert!(Arc::ptr_eq(&runtime.object_store(&url).unwrap(), expected));
1531+
}
1532+
for (key, _) in &seeded {
1533+
object_store_cache().write().unwrap().remove(key);
1534+
}
1535+
}
1536+
13961537
/// Checks that native file construction returns Local and cached Hadoop file routing returns
13971538
/// Other, with distinct registered stores. Removes its synthetic Hadoop cache entry on success.
13981539
#[test]

0 commit comments

Comments
 (0)