Skip to content

Commit d36ad9f

Browse files
committed
fix: escape the output path before parsing it as a URI
The output path reaches both write serdes as `Path.toString`, which decodes percent escapes: a directory containing a space or a literal `%` yields a string that is not a valid URI, and `URI.create` throws on it. On Spark 4.0+ that exception escaped `CometWriteFiles.convert` and failed the query. On 3.x `CometDataWritingCommand.convert` caught it and silently handed the write back to Spark, so the native writer was never used for those paths. Round-trip through `Path` instead, which re-escapes. Only the scheme and authority reach `extractObjectStoreOptions`, but parsing has to succeed to get at them. Also assert that the INSERT INTO visibility test's write actually went native, rather than inferring it from the read-back.
1 parent df96895 commit d36ad9f

3 files changed

Lines changed: 42 additions & 7 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/operator/CometDataWritingCommand.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
package org.apache.comet.serde.operator
2121

22-
import java.net.URI
2322
import java.util.Locale
2423

2524
import scala.jdk.CollectionConverters._
@@ -129,8 +128,11 @@ object CometDataWritingCommand extends CometOperatorSerde[DataWritingCommandExec
129128
// Collect S3/cloud storage configurations
130129
val session = op.session
131130
val hadoopConf = session.sessionState.newHadoopConfWithOptions(cmd.options)
131+
// `outputPath` is `Path.toString`, which is not a valid URI string: it leaves spaces and
132+
// literal `%` unescaped, so `URI.create` would throw (and the catch below would silently
133+
// give the write back to Spark). Going through `Path` escapes them again.
132134
val objectStoreOptions =
133-
NativeConfig.extractObjectStoreOptions(hadoopConf, URI.create(outputPath))
135+
NativeConfig.extractObjectStoreOptions(hadoopConf, cmd.outputPath.toUri)
134136
objectStoreOptions.foreach { case (key, value) =>
135137
writerOpBuilder.putObjectStoreOptions(key, value)
136138
}

‎spark/src/main/scala/org/apache/comet/serde/operator/CometWriteFiles.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919

2020
package org.apache.comet.serde.operator
2121

22-
import java.net.URI
2322
import java.util.Locale
2423

24+
import org.apache.hadoop.fs.Path
2525
import org.apache.parquet.hadoop.ParquetOutputFormat
2626
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
2727
import org.apache.spark.sql.comet.{CometNativeExec, CometWriteFilesExec}
@@ -141,8 +141,12 @@ object CometWriteFiles extends CometOperatorSerde[WriteFilesExec] {
141141
// getSupportLevel already declined the write if the tag is absent, so this cannot be empty.
142142
outputPathOf(op).foreach { outputPath =>
143143
val hadoopConf = op.session.sessionState.newHadoopConfWithOptions(op.options)
144+
// The tag holds `Path.toString`, which is not a valid URI string: it leaves spaces and
145+
// literal `%` unescaped, so `URI.create` would throw. Round-tripping through `Path` escapes
146+
// them again. Only the scheme and authority matter to `extractObjectStoreOptions`, but
147+
// parsing has to succeed to get at them.
144148
NativeConfig
145-
.extractObjectStoreOptions(hadoopConf, URI.create(outputPath))
149+
.extractObjectStoreOptions(hadoopConf, new Path(outputPath).toUri)
146150
.foreach { case (key, value) => writerOpBuilder.putObjectStoreOptions(key, value) }
147151
}
148152

‎spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -865,6 +865,27 @@ class CometParquetWriterSuite extends CometTestBase {
865865
}
866866
}
867867

868+
test("output path needing URI escaping still writes natively") {
869+
// The output path reaches the serde as `Path.toString`, which leaves characters that are
870+
// illegal in a URI unescaped. `URI.create` then throws for a space or a literal `%`, costing
871+
// the write its native path - silently on Spark 3.x, where the serde catches the failure and
872+
// hands the write back to Spark.
873+
Seq("dir with space", "dir%with%percent", "dir with space and %25").foreach { dirName =>
874+
withTempPath { dir =>
875+
val outputPath = new File(new File(dir, dirName), "output.parquet").getAbsolutePath
876+
val sourcePath = new File(dir, "source.parquet").getAbsolutePath
877+
withNativeWriter {
878+
val df = materializeAsCometSource(
879+
(1 to 100).map(i => (i, s"str_$i")).toDF("id", "name"),
880+
sourcePath)
881+
val plan = captureWritePlan(p => df.write.parquet(p), outputPath)
882+
assertHasCometNativeWriteExec(plan)
883+
checkAnswer(spark.read.parquet(outputPath), df)
884+
}
885+
}
886+
}
887+
}
888+
868889
// ---------------------------------------------------------------------------------------------
869890
// Spark 4.0+ only. These cover behavior that comes from leaving Spark's write framework in
870891
// place, which is only possible where `V1WritesUtils.getWriteFilesOpt` matches the
@@ -937,7 +958,11 @@ class CometParquetWriterSuite extends CometTestBase {
937958
sql("INSERT INTO comet_write_source VALUES (1, 'a'), (2, 'b')")
938959
}
939960
withNativeWriter {
940-
sql("INSERT INTO comet_write_target SELECT id, name FROM comet_write_source")
961+
// Assert the write itself went native: a fallback to Spark's writer would make the
962+
// read-back pass for the wrong reason.
963+
assertHasCometNativeWriteExec(
964+
captureWritePlan(
965+
sql("INSERT INTO comet_write_target SELECT id, name FROM comet_write_source")))
941966
}
942967
checkAnswer(spark.table("comet_write_target"), Row(1L, "a") :: Row(2L, "b") :: Nil)
943968
}
@@ -1172,7 +1197,11 @@ class CometParquetWriterSuite extends CometTestBase {
11721197
* @return
11731198
* The captured execution plan
11741199
*/
1175-
private def captureWritePlan(writeOp: String => Unit, outputPath: String): SparkPlan = {
1200+
private def captureWritePlan(writeOp: String => Unit, outputPath: String): SparkPlan =
1201+
captureWritePlan(writeOp(outputPath))
1202+
1203+
/** As above, for a write that names its own target (an `INSERT INTO`, for example). */
1204+
private def captureWritePlan(writeOp: => Unit): SparkPlan = {
11761205
var capturedPlan: Option[QueryExecution] = None
11771206

11781207
val listener = new org.apache.spark.sql.util.QueryExecutionListener {
@@ -1191,7 +1220,7 @@ class CometParquetWriterSuite extends CometTestBase {
11911220
spark.listenerManager.register(listener)
11921221

11931222
try {
1194-
writeOp(outputPath)
1223+
writeOp
11951224

11961225
// Wait for listener to be called with timeout
11971226
val maxWaitTimeMs = 15000

0 commit comments

Comments
 (0)