Skip to content

Commit a4fa4ab

Browse files
committed
feat: support S3 compliant filesystems
1 parent 3db38b9 commit a4fa4ab

13 files changed

Lines changed: 343 additions & 162 deletions

File tree

‎native/core/src/execution/operators/iceberg_scan.rs‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,10 @@ impl IcebergScanExec {
255255
};
256256
match scheme {
257257
"file" => Ok(Arc::new(OpenDalStorageFactory::Fs)),
258-
"s3" | "s3a" | "blob" => {
258+
// blob:// never reaches here: the planner normalizes it to s3:// via
259+
// s3_blob_fs_support::normalize_object_store_url_string before the scan is built. s3a
260+
// passes through unchanged (not an s3-compliant alias on the Iceberg path), so keep it.
261+
"s3" | "s3a" => {
259262
let customized_credential_load =
260263
build_s3_credential_loader(path, catalog_properties, catalog_name);
261264
Ok(Arc::new(OpenDalStorageFactory::S3 {
@@ -371,15 +374,16 @@ impl IcebergScanExec {
371374
// Object-store's AmazonS3Builder defaults the SigV4 region to `us-east-1` when unset;
372375
// iceberg-storage-opendal's S3 factory instead errors with `region is missing. Please
373376
// find it by S3::detect_region() or set them in env.` Non-AWS S3-compliant storage
374-
// services accept any region in the credential, so default to `us-east-1` when the
375-
// catalog didn't ship one. Both key spellings iceberg-rust reads (`client.region`
376-
// wins over `s3.region`).
377+
// services accept any region in the credential, so default to `us-east-1` -- but ONLY when
378+
// neither the catalog NOR the AWS environment supplies a region. opendal/reqsign reads
379+
// `AWS_REGION` / `AWS_DEFAULT_REGION`, so forcing `us-east-1` unconditionally would
380+
// override an `AWS_REGION=us-west-2` and break auth for AWS buckets outside us-east-1.
381+
// Both catalog key spellings iceberg-rust reads (`client.region` wins over `s3.region`).
377382
let region_forwarded = catalog_properties.contains_key("s3.region")
378-
|| catalog_properties.contains_key("client.region");
379-
let is_s3_family = matches!(
380-
metadata_location.split_once("://"),
381-
Some(("s3" | "s3a" | "blob", _))
382-
);
383+
|| catalog_properties.contains_key("client.region")
384+
|| env_region_present();
385+
// blob:// is already normalized to s3:// upstream (s3_blob_fs_support); s3a passes through.
386+
let is_s3_family = matches!(metadata_location.split_once("://"), Some(("s3" | "s3a", _)));
383387
if !region_forwarded && is_s3_family {
384388
file_io_builder = file_io_builder.with_prop("s3.region", "us-east-1");
385389
}
@@ -390,6 +394,17 @@ impl IcebergScanExec {
390394

391395
const STORAGE_PROPERTY_PREFIXES: &[&str] = &["s3.", "gcs.", "adls.", "client."];
392396

397+
/// True if the AWS environment supplies a region (`AWS_REGION` / `AWS_DEFAULT_REGION`).
398+
///
399+
/// opendal's S3 service reads these via reqsign, so Comet must not clobber them with a forced
400+
/// `us-east-1` default when the catalog omits a region (see the region defaulting in
401+
/// `load_file_io`).
402+
fn env_region_present() -> bool {
403+
["AWS_REGION", "AWS_DEFAULT_REGION"]
404+
.iter()
405+
.any(|k| std::env::var(k).is_ok_and(|v| !v.is_empty()))
406+
}
407+
393408
/// Wires the configured Comet credential provider into opendal's S3 service, or returns `None`
394409
/// so opendal falls back to its default credential chain.
395410
fn build_s3_credential_loader(

‎native/core/src/execution/planner.rs‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ use iceberg::expr::Bind;
9595
use crate::execution::operators::ExecutionError::GeneralError;
9696
use crate::execution::shuffle::{CometPartitioning, CompressionCodec};
9797
use crate::execution::spark_plan::SparkPlan;
98-
use crate::parquet::objectstore::blob_alias::{
98+
use crate::parquet::objectstore::s3_blob_fs_support::{
9999
normalize_object_store_url, normalize_object_store_url_string,
100100
};
101101
use crate::parquet::parquet_support::prepare_object_store_with_configs;
@@ -4059,7 +4059,11 @@ fn parse_file_scan_tasks_from_common(
40594059
};
40604060

40614061
Ok(iceberg::scan::FileScanTaskDeleteFile {
4062-
// Normalize blob/s3a aliases so iceberg-rust routes through S3, not LocalFs.
4062+
// Delete-file path: safe to normalize the blob alias to s3:// (unlike
4063+
// `data_file_path`). This path only locates the delete file for fetching; it is
4064+
// never matched against a data-file identity. Normalizing also lets the
4065+
// single-slash `blob:/bucket/...` form resolve, and the normalizer preserves the
4066+
// raw object key so opendal does not double-encode it.
40634067
file_path: normalize_object_store_url_string(&del.file_path)?,
40644068
file_type,
40654069
// Not serialized; filled in by IcebergScanExec::fill_delete_file_sizes.
@@ -4281,8 +4285,13 @@ fn parse_file_scan_tasks_from_common(
42814285

42824286
Ok(iceberg::scan::FileScanTask {
42834287
file_size_in_bytes: proto_task.file_size_in_bytes,
4284-
// Normalize blob/s3a aliases so iceberg-rust routes through S3, not LocalFs.
4285-
data_file_path: normalize_object_store_url_string(&proto_task.data_file_path)?,
4288+
// Keep the RAW data-file path -- do NOT rewrite blob:// to s3://. iceberg-rust
4289+
// matches positional deletes by comparing this string against the data-file path
4290+
// recorded inside the delete file (as Iceberg wrote it), so changing the scheme
4291+
// here silently drops deletes. The S3 backend opens a raw blob:// path fine (it
4292+
// takes the bucket from the host and slices the key out verbatim), and
4293+
// `metadata_location` (normalized above) already selected the S3 storage factory.
4294+
data_file_path: proto_task.data_file_path.clone(),
42864295
start: proto_task.start,
42874296
length: proto_task.length,
42884297
record_count: proto_task.record_count,

‎native/core/src/lib.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,18 +171,23 @@ pub extern "system" fn Java_org_apache_comet_NativeBase_isFeatureEnabled(
171171
/// schemes are handled separately on the JVM side via the user's libhdfs scheme config.)
172172
///
173173
/// `blob` is not recognized by `ObjectStoreScheme::parse`, but Comet treats it as a synonym
174-
/// for `s3` (rewritten in `prepare_object_store_with_configs`), so report it as supported.
174+
/// for `s3` (rewritten in `prepare_object_store_with_configs`), so report it as supported. The
175+
/// alias set lives in `s3_blob_fs_support::is_s3_compliant_alias_scheme`, not hardcoded here.
175176
#[no_mangle]
176177
pub extern "system" fn Java_org_apache_comet_NativeBase_isObjectStoreSchemeSupported(
177178
env: EnvUnowned,
178179
_: JClass,
179180
url: JString,
180181
) -> jni::sys::jboolean {
181182
try_unwrap_or_throw(&env, |env| {
183+
use crate::parquet::objectstore::s3_blob_fs_support::is_s3_compliant_alias_scheme;
182184
let url_str: String = url.try_to_string(env)?;
183185
let supported = url::Url::parse(&url_str)
184186
.ok()
185-
.map(|u| u.scheme() == "blob" || object_store::ObjectStoreScheme::parse(&u).is_ok())
187+
.map(|u| {
188+
is_s3_compliant_alias_scheme(u.scheme())
189+
|| object_store::ObjectStoreScheme::parse(&u).is_ok()
190+
})
186191
.unwrap_or(false);
187192
Ok(supported)
188193
})

‎native/core/src/parquet/objectstore/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@
1616
// under the License.
1717

1818
pub mod azure;
19-
pub mod blob_alias;
2019
pub mod s3;
20+
pub mod s3_blob_fs_support;

native/core/src/parquet/objectstore/blob_alias.rs renamed to native/core/src/parquet/objectstore/s3_blob_fs_support.rs

Lines changed: 62 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ pub(crate) fn normalize_object_store_url(
6565
// `file:/x` -> `file:///x` round-trip that would break iceberg delete matching is harmless
6666
// here. See `normalize_object_store_url_string` for why the Iceberg path is narrower.
6767
let scheme = url.scheme();
68-
if scheme != "s3a" && scheme != "blob" {
68+
if scheme != "s3a" && !is_s3_compliant_alias_scheme(scheme) {
6969
return Ok(url);
7070
}
7171
rewrite_alias_to_s3(url)
@@ -80,6 +80,18 @@ pub(crate) fn normalize_object_store_url(
8080
/// schemes that genuinely need an `s3://` shape to reach an object store belong here.
8181
const S3_COMPLIANT_ALIAS_SCHEMES: &[&str] = &["blob"];
8282

83+
/// True if `scheme` is one of the s3-compliant alias schemes Comet rewrites to canonical `s3://`
84+
/// ([`S3_COMPLIANT_ALIAS_SCHEMES`], currently only `blob`).
85+
///
86+
/// These are exactly the schemes that (a) Comet routes to an `object_store::AmazonS3` store and
87+
/// (b) `object_store::ObjectStoreScheme::parse` does NOT recognize on its own. Callers outside
88+
/// this module (the JVM scheme-support gate in `lib.rs`) ask here instead of hardcoding the
89+
/// literal, so blob knowledge stays in one home. `s3a` is deliberately excluded: `object_store`
90+
/// already recognizes it, so it needs no special-casing at the support gate.
91+
pub(crate) fn is_s3_compliant_alias_scheme(scheme: &str) -> bool {
92+
S3_COMPLIANT_ALIAS_SCHEMES.contains(&scheme)
93+
}
94+
8395
/// Rewrites an already-parsed alias URL (`blob`/`s3a`) to canonical `s3://bucket/key`, promoting
8496
/// the first path segment into the host for the single-slash `blob:/bucket/key` (Java opaque
8597
/// form) and three-slash `blob:///bucket/key` (empty authority) shapes that report host=None.
@@ -111,47 +123,71 @@ fn rewrite_alias_to_s3(mut url: Url) -> Result<Url, ExecutionError> {
111123
Ok(url)
112124
}
113125

114-
/// String-returning normalizer for the Iceberg path, where paths are handed to iceberg-rust as
115-
/// owned strings on `FileScanTask` / `FileScanTaskDeleteFile`.
126+
/// String-returning normalizer for the Iceberg path. Used for `metadata_location` (storage-access
127+
/// config: it selects the storage factory and region and is never opened as an object) and for
128+
/// delete-file paths (opened, but never matched against a data-file identity). `data_file_path` is
129+
/// deliberately NOT passed through here -- see `parse_file_scan_tasks_from_common`, which hands it
130+
/// to iceberg-rust raw so positional deletes still match.
116131
///
117-
/// Unlike the Url-returning [`normalize_object_store_url`], this MUST return every non-`blob`
118-
/// input byte-for-byte. iceberg-rust keys its positional/equality delete maps by the delete
119-
/// file's recorded `file_path` column (raw, as Iceberg Java wrote it) and looks them up by the
120-
/// `data_file_path` Comet supplies here. Any rewrite Comet applies to `data_file_path` that the
121-
/// delete file's stored path did not get -- `s3a://` -> `s3://`, or `url`'s `file:/x` ->
122-
/// `file:///x` re-serialization -- desyncs the two and silently drops the deletes (observed as a
123-
/// merge-on-read table returning rows that should have been deleted, and rewrites failing to mark
124-
/// data files deleted).
132+
/// Two invariants:
125133
///
126-
/// So we only rewrite the s3-compliant alias schemes that genuinely need an `s3://` shape to
127-
/// reach an object store ([`S3_COMPLIANT_ALIAS_SCHEMES`], currently just `blob`). `file://`,
128-
/// `s3://`, `s3a://`, and schemeless bare paths (a local Hadoop-catalog warehouse) all pass
129-
/// through unchanged; iceberg-storage-opendal's `storage_factory_for` routes a `blob:/...` form
130-
/// correctly only after this rewrite and everything else via its own scheme detection.
134+
/// 1. Every non-alias input is returned byte-for-byte (`file://`, `s3://`, `s3a://`, schemeless
135+
/// bare paths). iceberg-rust keys its positional/equality delete maps by the delete file's
136+
/// recorded `file_path` column and looks them up by the `data_file_path` Comet supplies, so any
137+
/// rewrite of a path also used as a delete-match key desyncs the two and silently drops deletes.
138+
///
139+
/// 2. For the alias schemes it DOES rewrite ([`S3_COMPLIANT_ALIAS_SCHEMES`], currently `blob`), the
140+
/// raw object-key bytes are preserved. iceberg-storage-opendal's S3 backend slices the key out
141+
/// of this string as `&path[prefix.len()..]` and hands it to opendal, which percent-encodes it
142+
/// once. Re-serializing through `url::Url` (`String::from(Url)`) would percent-encode the key
143+
/// here first (`café` -> `caf%C3%A9`) and opendal would encode the `%` again (`caf%25C3%25A9`),
144+
/// targeting the wrong object. So the scheme is swapped on the ORIGINAL bytes, never via `Url`.
131145
pub(crate) fn normalize_object_store_url_string(path: &str) -> Result<String, ExecutionError> {
132146
let Ok(url) = Url::parse(path) else {
133147
// Not a URL: a schemeless bare path from a local Hadoop-catalog warehouse (e.g.
134148
// `/tmp/warehouse/db/t/metadata/v1.metadata.json` from `metadataFileLocation()`). Hand it
135-
// to iceberg-storage-opendal's LocalFs backend unchanged, as the raw-passthrough these
136-
// call sites used before blob normalization did.
149+
// to iceberg-storage-opendal's LocalFs backend unchanged.
137150
return Ok(path.to_string());
138151
};
139-
if !S3_COMPLIANT_ALIAS_SCHEMES.contains(&url.scheme()) {
140-
// file://, s3://, s3a://, ... -> return the ORIGINAL string byte-for-byte. Re-serializing
141-
// through `url` (e.g. `String::from(url)`) rewrites the single-slash `file:/x` to the
142-
// three-slash `file:///x`, which no longer matches the delete file's stored path.
152+
let scheme = url.scheme();
153+
if !is_s3_compliant_alias_scheme(scheme) {
154+
// file://, s3://, s3a://, ... -> the ORIGINAL string, byte-for-byte.
143155
return Ok(path.to_string());
144156
}
145-
// A recognized s3-compliant alias (blob) -> canonical `s3://bucket/key`, promoting the bucket
146-
// out of the path for the single/three-slash forms. `String::from(Url)` moves the Url's
147-
// serialization buffer out rather than re-serializing through `Display`.
148-
Ok(String::from(rewrite_alias_to_s3(url)?))
157+
// Strip `blob:` and every leading slash, then prepend `s3://`. This handles all three shapes
158+
// uniformly -- `blob://bucket/key`, the Java opaque `blob:/bucket/key`, and the empty-authority
159+
// `blob:///bucket/key` -- promoting the first path segment into the authority. Operating on the
160+
// ORIGINAL bytes (not `url`) keeps the raw object key intact so opendal does not double-encode
161+
// it (invariant 2 above); `scheme.len()` is ASCII, so it indexes `path` correctly.
162+
let rest = path[scheme.len() + 1..].trim_start_matches('/');
163+
if rest.is_empty() {
164+
return Err(ExecutionError::GeneralError(format!(
165+
"{scheme}:// URL is missing bucket name: {path}"
166+
)));
167+
}
168+
Ok(format!("s3://{rest}"))
149169
}
150170

151171
#[cfg(test)]
152172
mod tests {
153173
use super::*;
154174

175+
#[test]
176+
fn test_normalize_object_store_url_string_preserves_raw_non_ascii_key() {
177+
// The key bytes must survive verbatim. Re-serializing through `url::Url`
178+
// (`String::from(Url)`) would percent-encode `café` to `caf%C3%A9`, which
179+
// iceberg-storage-opendal's S3 backend then re-encodes to `caf%25C3%25A9`, hitting the
180+
// wrong object. The scheme swap must operate on the raw bytes and leave the key untouched.
181+
let out =
182+
normalize_object_store_url_string("blob://bucket/warehouse/café/part.parquet").unwrap();
183+
assert_eq!(out, "s3://bucket/warehouse/café/part.parquet");
184+
185+
// The single-slash form promotes the bucket into the authority but still keeps the raw key.
186+
let out =
187+
normalize_object_store_url_string("blob:/bucket/warehouse/café/part.parquet").unwrap();
188+
assert_eq!(out, "s3://bucket/warehouse/café/part.parquet");
189+
}
190+
155191
#[test]
156192
fn test_normalize_object_store_url_string_blob_single_slash_form() {
157193
// Java's URI.toString() collapses `blob:///bucket/key` to `blob:/bucket/key` (opaque form

‎native/core/src/parquet/parquet_support.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,7 @@ pub(crate) fn prepare_object_store_with_configs(
526526
url: String,
527527
object_store_configs: &HashMap<String, String>,
528528
) -> Result<(ObjectStoreUrl, Path), ExecutionError> {
529-
let url = super::objectstore::blob_alias::normalize_object_store_url(
529+
let url = super::objectstore::s3_blob_fs_support::normalize_object_store_url(
530530
url.as_str(),
531531
object_store_configs,
532532
)?;

‎spark/src/main/java/org/apache/comet/parquet/CometFileKeyUnwrapper.java‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import java.util.concurrent.ConcurrentHashMap;
2323

24+
import org.apache.commons.lang3.StringUtils;
2425
import org.apache.hadoop.conf.Configuration;
2526
import org.apache.hadoop.fs.Path;
2627
import org.apache.parquet.crypto.DecryptionKeyRetriever;
@@ -92,6 +93,12 @@
9293
*/
9394
public class CometFileKeyUnwrapper {
9495

96+
// S3-family schemes that all address the same logical filesystem and must fold to a single
97+
// canonical cache key. Kept in sync with the native alias handling
98+
// (native/core/src/parquet/objectstore/s3_blob_fs_support.rs and the s3a/s3n aliases object_store
99+
// recognizes). The `:` disambiguates them, so order is irrelevant.
100+
private static final String[] S3_ALIAS_SCHEMES = {"s3", "s3n", "s3a", "blob"};
101+
95102
// Each file path gets a unique DecryptionKeyRetriever
96103
private final ConcurrentHashMap<String, DecryptionKeyRetriever> retrieverCache =
97104
new ConcurrentHashMap<>();
@@ -109,21 +116,25 @@ public class CometFileKeyUnwrapper {
109116
* native side JNIs back with the scheme after `prepare_object_store_with_configs` has already
110117
* rewritten aliases to s3://.
111118
*
119+
* <p>The native rewrite also promotes the first path segment into the authority for the
120+
* single-slash `blob:/bucket/key` (Java opaque form) and triple-slash `blob:///bucket/key` (empty
121+
* authority) shapes, so the get side reconstructs `s3://bucket/key`. This method must apply the
122+
* same promotion, otherwise a `blob:/...` or `blob:///...` input listed by Spark would be cached
123+
* under a key that never matches the get side's `s3a://bucket/key`.
124+
*
112125
* @param filePath The file path that may contain an S3 URI
113-
* @return The file path with normalized S3 scheme (s3a://)
126+
* @return The file path with normalized S3 scheme (s3a://) and bucket in the authority
114127
*/
115128
private String normalizeS3Scheme(final String filePath) {
116-
// Normalize s3://, s3n://, and blob:// to s3a:// for consistent cache lookups
117-
// This handles the case where ObjectStoreUrl uses s3:// but Spark uses s3a:// or blob://
118-
String s3Prefix = "s3://";
119-
String s3nPrefix = "s3n://";
120-
String blobPrefix = "blob://";
121-
if (filePath.startsWith(s3Prefix)) {
122-
return "s3a://" + filePath.substring(s3Prefix.length());
123-
} else if (filePath.startsWith(s3nPrefix)) {
124-
return "s3a://" + filePath.substring(s3nPrefix.length());
125-
} else if (filePath.startsWith(blobPrefix)) {
126-
return "s3a://" + filePath.substring(blobPrefix.length());
129+
// Normalize s3://, s3n://, s3a://, and blob:// (any of one/two/three slashes) to a canonical
130+
// s3a://<bucket>/<key> for consistent cache lookups. Stripping every leading slash after the
131+
// scheme and re-joining with "//" promotes the first path segment to the authority for the
132+
// single-/triple-slash forms, matching the native rewrite_alias_to_s3 behavior.
133+
for (String scheme : S3_ALIAS_SCHEMES) {
134+
String prefix = scheme + ":";
135+
if (filePath.startsWith(prefix)) {
136+
return "s3a://" + StringUtils.stripStart(filePath.substring(prefix.length()), "/");
137+
}
127138
}
128139
return filePath;
129140
}

0 commit comments

Comments
 (0)