Skip to content

Commit 032c2a2

Browse files
committed
feat: support S3 compliant filesystems
1 parent 1fea583 commit 032c2a2

11 files changed

Lines changed: 172 additions & 296 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1769,11 +1769,9 @@ impl PhysicalPlanner {
17691769
.iter()
17701770
.map(|(k, v)| (k.clone(), v.clone()))
17711771
.collect();
1772-
// Rewrite opted-in S3-compliant alias schemes (`fs.comet.s3Compliant.schemes`,
1773-
// carried in catalog_properties) to canonical `s3://bucket/key`, so
1774-
// `storage_factory_for` routes to the S3 backend instead of the LocalFs fallback it
1775-
// picks for schemes without a `://` authority. The single-slash `blob:/bucket/key`
1776-
// form is handled defensively; real object-store metadata locations are two-slash.
1772+
// Rewrite opted-in aliases (carried in catalog_properties) to `s3://bucket/key` so
1773+
// `storage_factory_for` routes to the S3 backend, not the LocalFs fallback it picks
1774+
// for schemes without a `://` authority. Single-slash form handled defensively.
17771775
let metadata_location = normalize_object_store_url_string(
17781776
&common.metadata_location,
17791777
&common.catalog_properties,
@@ -4059,11 +4057,9 @@ fn parse_file_scan_tasks_from_common(
40594057
};
40604058

40614059
Ok(iceberg::scan::FileScanTaskDeleteFile {
4062-
// Delete-file path: safe to rewrite an opted-in alias to s3:// (unlike
4063-
// `data_file_path`), since it only locates the delete file for fetching and is
4064-
// never matched against a data-file identity. The normalizer preserves the raw
4065-
// object key so opendal does not double-encode it, and handles the single-slash
4066-
// form defensively.
4060+
// Safe to rewrite an opted-in alias here (unlike `data_file_path`): this only
4061+
// locates the delete file, never matched against a data-file identity. The
4062+
// normalizer preserves the raw key so opendal does not double-encode it.
40674063
file_path: normalize_object_store_url_string(
40684064
&del.file_path,
40694065
&proto_common.catalog_properties,
@@ -4288,12 +4284,11 @@ fn parse_file_scan_tasks_from_common(
42884284

42894285
Ok(iceberg::scan::FileScanTask {
42904286
file_size_in_bytes: proto_task.file_size_in_bytes,
4291-
// Keep the RAW data-file path -- do NOT rewrite blob:// to s3://. iceberg-rust
4292-
// matches positional deletes by comparing this string against the data-file path
4293-
// recorded inside the delete file (as Iceberg wrote it), so changing the scheme
4294-
// here silently drops deletes. The S3 backend opens a raw blob:// path fine (it
4295-
// takes the bucket from the host and slices the key out verbatim), and
4296-
// `metadata_location` (normalized above) already selected the S3 storage factory.
4287+
// RAW data-file path -- do NOT rewrite the alias to s3://. iceberg-rust matches
4288+
// positional deletes by comparing this against the path recorded inside the delete
4289+
// file, so changing the scheme drops deletes. The S3 backend opens a raw alias path
4290+
// fine (bucket from host, key sliced verbatim), and `metadata_location` above
4291+
// already selected the storage factory.
42974292
data_file_path: proto_task.data_file_path.clone(),
42984293
start: proto_task.start,
42994294
length: proto_task.length,

‎native/core/src/lib.rs‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -160,20 +160,16 @@ pub extern "system" fn Java_org_apache_comet_NativeBase_isFeatureEnabled(
160160
})
161161
}
162162

163-
/// JNI method: does object_store recognize this URL's scheme?
163+
/// JNI: does object_store recognize this URL's scheme?
164164
///
165-
/// This is the source of truth for the JVM planner's "can Comet's native reader handle this
166-
/// filesystem?" check. Comet's `prepare_object_store_with_configs` dispatches non-hdfs/non-s3
167-
/// schemes to object_store's `parse_url`, which is driven by `ObjectStoreScheme::parse`; an
168-
/// unrecognized scheme (e.g. a custom Hadoop FileSystem) fails there at execution time. By
169-
/// answering from `ObjectStoreScheme::parse` here, the planner can decline early without
170-
/// hardcoding -- and drifting from -- the object_store-supported scheme set. (hdfs / libhdfs
171-
/// schemes are handled separately on the JVM side via the user's libhdfs scheme config.)
165+
/// Source of truth for the JVM planner's "can the native reader handle this filesystem?" check.
166+
/// `prepare_object_store_with_configs` dispatches non-hdfs/non-s3 schemes to object_store's
167+
/// `parse_url` (driven by `ObjectStoreScheme::parse`), so answering from the same parser lets the
168+
/// planner decline early without hardcoding the supported set. (hdfs/libhdfs are handled JVM-side.)
172169
///
173-
/// Alias schemes such as `blob` are NOT answered here: this gate has no config, so it cannot
174-
/// know which schemes the operator opted into. That opt-in is decided in the JVM `CometScanRule`
175-
/// via `fs.comet.s3Compliant.schemes`; opted-in schemes are rewritten to `s3://` before reaching
176-
/// `prepare_object_store_with_configs`.
170+
/// Alias schemes (e.g. `blob`) are NOT answered here: this gate has no config. Their opt-in lives
171+
/// in the JVM `CometScanRule` via `fs.comet.s3Compliant.schemes`, and opted-in schemes are
172+
/// rewritten to `s3://` before reaching `prepare_object_store_with_configs`.
177173
#[no_mangle]
178174
pub extern "system" fn Java_org_apache_comet_NativeBase_isObjectStoreSchemeSupported(
179175
env: EnvUnowned,

0 commit comments

Comments
 (0)