@@ -32,6 +32,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, ScalarSubquery => Exe
3232import org .apache .spark .sql .execution .datasources .{FilePartition , PartitionedFile }
3333import org .apache .spark .sql .types .{ByteType , LongType , MetadataBuilder , StructField , StructType }
3434
35+ import org .apache .comet .CometSparkSessionExtensions .withFallbackReason
3536import org .apache .comet .objectstore .NativeConfig
3637import org .apache .comet .serde .OperatorOuterClass
3738import org .apache .comet .serde .OperatorOuterClass .Operator
@@ -117,7 +118,8 @@ object CometDeltaNativeScan
117118 /**
118119 * Build the planning-time `DeltaScan` operator (common data only; file partitions are injected
119120 * lazily at execution). Returns None when an output data type cannot be serialized or the plan
120- * shape is not one we can translate faithfully. `memo` is the same claim-memo instance
121+ * shape is not one we can translate faithfully, and tags `scanExec` with the reason when the
122+ * data files need more than one ABFS store. `memo` is the same claim-memo instance
121123 * [[DeltaScanSupport.declineReason ]] populated on this claim; its `hadoopConf` and
122124 * `dvDescriptors` are reused here rather than recomputed.
123125 */
@@ -127,13 +129,21 @@ object CometDeltaNativeScan
127129 memo : DeltaScanSupport .DeltaClaimMemo ): Option [Operator ] = {
128130 val relation = scanExec.relation
129131
130- val firstFileUri = scanHelper.selectedPartitions
131- .flatMap(_.files.headOption)
132- .headOption
133- .map(_.getPath.toUri)
132+ val dataFileUris = scanHelper.selectedPartitions.view.flatMap(_.files).map(_.getPath.toUri)
133+ val firstFileUri = dataFileUris.headOption
134134
135135 val hadoopConf = memo.hadoopConf
136136
137+ val objectStoreOptions =
138+ NativeConfig .extractScanObjectStoreOptions(hadoopConf, dataFileUris) match {
139+ case Right (options) => options
140+ case Left (reason) =>
141+ // declineReason already declines data files on more than one store; this only
142+ // catches ABFS authorities that differ in case or percent-encoding alone.
143+ withFallbackReason(scanExec, reason)
144+ return None
145+ }
146+
137147 val tableRootPath = relation.location.rootPaths.head
138148 val tableRoot = tableRootPath.toString
139149
@@ -148,11 +158,11 @@ object CometDeltaNativeScan
148158 partitionSchema = toPhysical(scanExec, relation.partitionSchema),
149159 fileConstantMetadataColumns = scanExec.fileConstantMetadataColumns,
150160 dataFilters = scanHelper.supportedDataFilters,
151- firstFileUri = firstFileUri ,
161+ objectStoreOptions = objectStoreOptions ,
152162 hadoopConf = hadoopConf,
153163 conf = scanExec.conf)
154164 } else {
155- buildDvScanCommon(scanExec, scanHelper, firstFileUri , hadoopConf)
165+ buildDvScanCommon(scanExec, scanHelper, objectStoreOptions , hadoopConf)
156166 }
157167
158168 commonOpt.map { commonBuilder =>
@@ -221,9 +231,12 @@ object CometDeltaNativeScan
221231
222232 /**
223233 * Unions `NativeConfig.extractObjectStoreOptions` over every `uris` authority. Safe to union
224- * rather than pick one: extracted keys are scheme-disjoint prefixes (`fs.s3a.*` vs
225- * `fs.azure.*`, ...), so options for different schemes never collide, and re-extracting the
226- * same scheme from two URIs is idempotent.
234+ * rather than pick one: S3 and GCS options sit under their own scheme's prefix, and
235+ * re-extracting the same scheme from two URIs is idempotent. Azure options are not scoped to an
236+ * account and overlap (WASB copies `fs.azure.*`; ABFS forwards `comet.azure.*` markers plus
237+ * global `fs.azure.*` keys), but only the table root can bring an Azure authority here:
238+ * `DeltaScanSupport.declineReason` declines Azure data files and deletion vectors, whose
239+ * container is userinfo.
227240 */
228241 private [delta] def mergedObjectStoreOptions (
229242 hadoopConf : org.apache.hadoop.conf.Configuration ,
@@ -388,7 +401,7 @@ object CometDeltaNativeScan
388401 private def buildDvScanCommon (
389402 scanExec : FileSourceScanExec ,
390403 scanHelper : CometScanExec ,
391- firstFileUri : Option [java.net. URI ],
404+ objectStoreOptions : Map [ String , String ],
392405 hadoopConf : org.apache.hadoop.conf.Configuration )
393406 : Option [OperatorOuterClass .NativeScanCommon .Builder ] = {
394407 val relation = scanExec.relation
@@ -482,7 +495,7 @@ object CometDeltaNativeScan
482495 CometNativeScan .populateScanConfFlags(
483496 commonBuilder,
484497 physicalRequired,
485- firstFileUri ,
498+ objectStoreOptions ,
486499 hadoopConf,
487500 scanExec.conf)
488501
0 commit comments