Skip to content

Commit deaf061

Browse files
committed
docs: clarify object-store cache and registration identity
1 parent 0ec209d commit deaf061

1 file changed

Lines changed: 8 additions & 0 deletions

File tree

‎native/core/src/parquet/parquet_support.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,10 @@ fn create_hdfs_object_store(
469469
})
470470
}
471471

472+
/// Cache identity: `(scheme://host:port, config_hash, hdfs_backend)`.
473+
/// Native `s3a` is normalized to `s3`; Hadoop-selected schemes keep their spelling.
474+
/// The hash covers the object-store configuration. The boolean is `true` for the
475+
/// Hadoop backend (including custom schemes routed through Hadoop), `false` for native.
472476
type ObjectStoreCacheKey = (String, u64, bool);
473477
type ObjectStoreCache = RwLock<HashMap<ObjectStoreCacheKey, Arc<dyn ObjectStore>>>;
474478

@@ -596,6 +600,10 @@ pub(crate) fn prepare_object_store_with_configs(
596600
ObjectStoreUrl::parse(url_key)?
597601
} else {
598602
let backend = if is_hdfs_scheme { "hdfs" } else { "native" };
603+
// DataFusion keys stores only by scheme and authority, so put configuration
604+
// and backend identity in the scheme while preserving the physical authority.
605+
// `+comet-` marks our internal registration suffix; encryption lookup strips
606+
// the complete suffix to recover the physical URI.
599607
ObjectStoreUrl::parse(format!(
600608
"{scheme}+comet-{config_hash:016x}-{backend}://{}",
601609
&url[url::Position::BeforeHost..url::Position::AfterPort],

0 commit comments

Comments
 (0)