@@ -65,7 +65,7 @@ pub(crate) fn normalize_object_store_url(
6565 // `file:/x` -> `file:///x` round-trip that would break iceberg delete matching is harmless
6666 // here. See `normalize_object_store_url_string` for why the Iceberg path is narrower.
6767 let scheme = url. scheme ( ) ;
68- if scheme != "s3a" && scheme != "blob" {
68+ if scheme != "s3a" && ! is_s3_compliant_alias_scheme ( scheme ) {
6969 return Ok ( url) ;
7070 }
7171 rewrite_alias_to_s3 ( url)
@@ -80,6 +80,18 @@ pub(crate) fn normalize_object_store_url(
8080/// schemes that genuinely need an `s3://` shape to reach an object store belong here.
8181const S3_COMPLIANT_ALIAS_SCHEMES : & [ & str ] = & [ "blob" ] ;
8282
83+ /// True if `scheme` is one of the s3-compliant alias schemes Comet rewrites to canonical `s3://`
84+ /// ([`S3_COMPLIANT_ALIAS_SCHEMES`], currently only `blob`).
85+ ///
86+ /// These are exactly the schemes that (a) Comet routes to an `object_store::AmazonS3` store and
87+ /// (b) `object_store::ObjectStoreScheme::parse` does NOT recognize on its own. Callers outside
88+ /// this module (the JVM scheme-support gate in `lib.rs`) ask here instead of hardcoding the
89+ /// literal, so blob knowledge stays in one home. `s3a` is deliberately excluded: `object_store`
90+ /// already recognizes it, so it needs no special-casing at the support gate.
91+ pub ( crate ) fn is_s3_compliant_alias_scheme ( scheme : & str ) -> bool {
92+ S3_COMPLIANT_ALIAS_SCHEMES . contains ( & scheme)
93+ }
94+
8395/// Rewrites an already-parsed alias URL (`blob`/`s3a`) to canonical `s3://bucket/key`, promoting
8496/// the first path segment into the host for the single-slash `blob:/bucket/key` (Java opaque
8597/// form) and three-slash `blob:///bucket/key` (empty authority) shapes that report host=None.
@@ -111,47 +123,71 @@ fn rewrite_alias_to_s3(mut url: Url) -> Result<Url, ExecutionError> {
111123 Ok ( url)
112124}
113125
114- /// String-returning normalizer for the Iceberg path, where paths are handed to iceberg-rust as
115- /// owned strings on `FileScanTask` / `FileScanTaskDeleteFile`.
126+ /// String-returning normalizer for the Iceberg path. Used for `metadata_location` (storage-access
127+ /// config: it selects the storage factory and region and is never opened as an object) and for
128+ /// delete-file paths (opened, but never matched against a data-file identity). `data_file_path` is
129+ /// deliberately NOT passed through here -- see `parse_file_scan_tasks_from_common`, which hands it
130+ /// to iceberg-rust raw so positional deletes still match.
116131///
117- /// Unlike the Url-returning [`normalize_object_store_url`], this MUST return every non-`blob`
118- /// input byte-for-byte. iceberg-rust keys its positional/equality delete maps by the delete
119- /// file's recorded `file_path` column (raw, as Iceberg Java wrote it) and looks them up by the
120- /// `data_file_path` Comet supplies here. Any rewrite Comet applies to `data_file_path` that the
121- /// delete file's stored path did not get -- `s3a://` -> `s3://`, or `url`'s `file:/x` ->
122- /// `file:///x` re-serialization -- desyncs the two and silently drops the deletes (observed as a
123- /// merge-on-read table returning rows that should have been deleted, and rewrites failing to mark
124- /// data files deleted).
132+ /// Two invariants:
125133///
126- /// So we only rewrite the s3-compliant alias schemes that genuinely need an `s3://` shape to
127- /// reach an object store ([`S3_COMPLIANT_ALIAS_SCHEMES`], currently just `blob`). `file://`,
128- /// `s3://`, `s3a://`, and schemeless bare paths (a local Hadoop-catalog warehouse) all pass
129- /// through unchanged; iceberg-storage-opendal's `storage_factory_for` routes a `blob:/...` form
130- /// correctly only after this rewrite and everything else via its own scheme detection.
134+ /// 1. Every non-alias input is returned byte-for-byte (`file://`, `s3://`, `s3a://`, schemeless
135+ /// bare paths). iceberg-rust keys its positional/equality delete maps by the delete file's
136+ /// recorded `file_path` column and looks them up by the `data_file_path` Comet supplies, so any
137+ /// rewrite of a path also used as a delete-match key desyncs the two and silently drops deletes.
138+ ///
139+ /// 2. For the alias schemes it DOES rewrite ([`S3_COMPLIANT_ALIAS_SCHEMES`], currently `blob`), the
140+ /// raw object-key bytes are preserved. iceberg-storage-opendal's S3 backend slices the key out
141+ /// of this string as `&path[prefix.len()..]` and hands it to opendal, which percent-encodes it
142+ /// once. Re-serializing through `url::Url` (`String::from(Url)`) would percent-encode the key
143+ /// here first (`café` -> `caf%C3%A9`) and opendal would encode the `%` again (`caf%25C3%25A9`),
144+ /// targeting the wrong object. So the scheme is swapped on the ORIGINAL bytes, never via `Url`.
131145pub ( crate ) fn normalize_object_store_url_string ( path : & str ) -> Result < String , ExecutionError > {
132146 let Ok ( url) = Url :: parse ( path) else {
133147 // Not a URL: a schemeless bare path from a local Hadoop-catalog warehouse (e.g.
134148 // `/tmp/warehouse/db/t/metadata/v1.metadata.json` from `metadataFileLocation()`). Hand it
135- // to iceberg-storage-opendal's LocalFs backend unchanged, as the raw-passthrough these
136- // call sites used before blob normalization did.
149+ // to iceberg-storage-opendal's LocalFs backend unchanged.
137150 return Ok ( path. to_string ( ) ) ;
138151 } ;
139- if !S3_COMPLIANT_ALIAS_SCHEMES . contains ( & url. scheme ( ) ) {
140- // file://, s3://, s3a://, ... -> return the ORIGINAL string byte-for-byte. Re-serializing
141- // through `url` (e.g. `String::from(url)`) rewrites the single-slash `file:/x` to the
142- // three-slash `file:///x`, which no longer matches the delete file's stored path.
152+ let scheme = url. scheme ( ) ;
153+ if !is_s3_compliant_alias_scheme ( scheme) {
154+ // file://, s3://, s3a://, ... -> the ORIGINAL string, byte-for-byte.
143155 return Ok ( path. to_string ( ) ) ;
144156 }
145- // A recognized s3-compliant alias (blob) -> canonical `s3://bucket/key`, promoting the bucket
146- // out of the path for the single/three-slash forms. `String::from(Url)` moves the Url's
147- // serialization buffer out rather than re-serializing through `Display`.
148- Ok ( String :: from ( rewrite_alias_to_s3 ( url) ?) )
157+ // Strip `blob:` and every leading slash, then prepend `s3://`. This handles all three shapes
158+ // uniformly -- `blob://bucket/key`, the Java opaque `blob:/bucket/key`, and the empty-authority
159+ // `blob:///bucket/key` -- promoting the first path segment into the authority. Operating on the
160+ // ORIGINAL bytes (not `url`) keeps the raw object key intact so opendal does not double-encode
161+ // it (invariant 2 above); `scheme.len()` is ASCII, so it indexes `path` correctly.
162+ let rest = path[ scheme. len ( ) + 1 ..] . trim_start_matches ( '/' ) ;
163+ if rest. is_empty ( ) {
164+ return Err ( ExecutionError :: GeneralError ( format ! (
165+ "{scheme}:// URL is missing bucket name: {path}"
166+ ) ) ) ;
167+ }
168+ Ok ( format ! ( "s3://{rest}" ) )
149169}
150170
151171#[ cfg( test) ]
152172mod tests {
153173 use super :: * ;
154174
175+ #[ test]
176+ fn test_normalize_object_store_url_string_preserves_raw_non_ascii_key ( ) {
177+ // The key bytes must survive verbatim. Re-serializing through `url::Url`
178+ // (`String::from(Url)`) would percent-encode `café` to `caf%C3%A9`, which
179+ // iceberg-storage-opendal's S3 backend then re-encodes to `caf%25C3%25A9`, hitting the
180+ // wrong object. The scheme swap must operate on the raw bytes and leave the key untouched.
181+ let out =
182+ normalize_object_store_url_string ( "blob://bucket/warehouse/café/part.parquet" ) . unwrap ( ) ;
183+ assert_eq ! ( out, "s3://bucket/warehouse/café/part.parquet" ) ;
184+
185+ // The single-slash form promotes the bucket into the authority but still keeps the raw key.
186+ let out =
187+ normalize_object_store_url_string ( "blob:/bucket/warehouse/café/part.parquet" ) . unwrap ( ) ;
188+ assert_eq ! ( out, "s3://bucket/warehouse/café/part.parquet" ) ;
189+ }
190+
155191 #[ test]
156192 fn test_normalize_object_store_url_string_blob_single_slash_form ( ) {
157193 // Java's URI.toString() collapses `blob:///bucket/key` to `blob:/bucket/key` (opaque form
0 commit comments