Skip to content

Commit 43bce31

Browse files
committed
refactor: address PR 5293 review feedback
- Decline the write when spark.sql.files.maxRecordsPerFile > 0 (option or conf), so users don't silently get a single oversized file where Spark would have rolled several. Matches Spark's option-over-conf precedence. - Note that the existing partitionColumns.nonEmpty guard also declines dynamic partition overwrite, since Spark only sets dynamicPartitionOverwrite when staticPartitions.size < partitionColumns.length. - Match Spark's SPARK-23271 behavior: if the child RDD has zero partitions, swap in a dummy single-partition RDD so the output still carries a schema-only file. Defensive - requiresNativeChildren rules out LocalTableScan today, but the swap costs nothing and preserves the semantics. - Add tests for the abort-and-retry path via an injected failing commit protocol, for the dynamic-overwrite fallback (checking untouched partitions survive), for the maxRecordsPerFile fallback (option and conf, verifying Spark's writer rolls 10 files), and for the schema-only empty-input write.
1 parent bcfef4b commit 43bce31

4 files changed

Lines changed: 225 additions & 7 deletions

File tree

‎docs/source/user-guide/latest/operators.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,9 @@ omitted from the tables below and may be reconsidered based on demand:
116116

117117
## Writes
118118

119-
| Operator | Status | Notes |
120-
| ---------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------- |
121-
| `WriteFilesExec` | ⚠️ | Experimental native Parquet writes, disabled by default (opt-in). Requires Spark 4.0+. Non-partitioned, non-bucketed writes only. |
119+
| Operator | Status | Notes |
120+
| ---------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
121+
| `WriteFilesExec` | ⚠️ | Experimental native Parquet writes, disabled by default (opt-in). Requires Spark 4.0+. Non-partitioned, non-bucketed writes only, and not when `spark.sql.files.maxRecordsPerFile` is set. |
122122

123123
## Python and UDF
124124

‎spark/src/main/scala/org/apache/comet/serde/operator/CometWriteFiles.scala‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import java.net.URI
2323
import java.util.Locale
2424

2525
import org.apache.parquet.hadoop.ParquetOutputFormat
26+
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
2627
import org.apache.spark.sql.comet.{CometNativeExec, CometWriteFilesExec}
2728
import org.apache.spark.sql.execution.datasources.WriteFilesExec
2829
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
@@ -85,9 +86,17 @@ object CometWriteFiles extends CometOperatorSerde[WriteFilesExec] {
8586
}
8687

8788
if (op.partitionColumns.nonEmpty || op.staticPartitions.nonEmpty) {
89+
// This also declines dynamic partition overwrite. `InsertIntoHadoopFsRelationCommand` only
90+
// sets `dynamicPartitionOverwrite` when `staticPartitions.size < partitionColumns.length`,
91+
// which implies partition columns, so a dynamic overwrite always lands here.
8892
return Unsupported(Some("Partitioned writes are not supported"))
8993
}
9094

95+
if (rollsFilesByRecordCount(op)) {
96+
return Unsupported(
97+
Some("Writes with spark.sql.files.maxRecordsPerFile set are not supported"))
98+
}
99+
91100
val codec = parseCompressionCodec(op)
92101
if (!supportedCompressionCodecs.contains(codec)) {
93102
return Unsupported(Some(s"Unsupported compression codec: $codec"))
@@ -162,6 +171,25 @@ object CometWriteFiles extends CometOperatorSerde[WriteFilesExec] {
162171
private def outputPathOf(op: WriteFilesExec): Option[String] =
163172
op.getTagValue(CometExecRule.WRITE_OUTPUT_PATH)
164173

174+
/**
175+
* Whether Spark would roll to a new file every N rows within a task.
176+
*
177+
* `SingleDirectoryDataWriter.write` rolls a new file once `maxRecordsPerFile` rows have been
178+
* written, incrementing the `-c$fileCounter%03d` suffix. Comet's writer asks the commit
179+
* protocol for one file per task and always writes `-c000`, so a task that should have produced
180+
* several files would produce one oversized file instead - a different layout than Spark's own
181+
* writer with no error. Decline the write rather than silently ignore the setting.
182+
*/
183+
private def rollsFilesByRecordCount(op: WriteFilesExec): Boolean = {
184+
// Same precedence as FileFormatWriter.write: the `maxRecordsPerFile` write option wins over
185+
// spark.sql.files.maxRecordsPerFile. Options are matched case-insensitively there.
186+
val maxRecordsPerFile = CaseInsensitiveMap(op.options)
187+
.get("maxRecordsPerFile")
188+
.map(_.toLong)
189+
.getOrElse(SQLConf.get.maxRecordsPerFile)
190+
maxRecordsPerFile > 0
191+
}
192+
165193
private def parseCompressionCodec(op: WriteFilesExec): String = {
166194
// `compression`, `parquet.compression` (i.e., ParquetOutputFormat.COMPRESSION), and
167195
// `spark.sql.parquet.compression.codec` are in order of precedence from highest to

‎spark/src/main/scala/org/apache/spark/sql/comet/CometWriteFilesExec.scala‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,17 @@ case class CometWriteFilesExec(
111111

112112
val childRDD = child.executeColumnar()
113113

114+
// SPARK-23271 (defensive): a zero-partition input would spawn no task and therefore write no
115+
// file at all, so the output directory would carry no schema for readers. Spark's own
116+
// WriteFilesExec swaps in a dummy single-partition RDD for exactly this case. In practice
117+
// CometWriteFiles.requiresNativeChildren rules out the sources (LocalTableScan) that produce
118+
// a zero-partition RDD, but the swap is kept to match Spark's semantics if that ever changes.
119+
val writeRDD = if (childRDD.getNumPartitions == 0) {
120+
sparkContext.parallelize(Seq.empty[ColumnarBatch], 1)
121+
} else {
122+
childRDD
123+
}
124+
114125
// Everything the write task needs is resolved here on the driver and captured by value. The
115126
// closure below must not touch `this`: a CometWriteFilesExec holds `nativeOp` plus the whole
116127
// converted child subtree, each node of which carries its own non-transient protobuf, so
@@ -124,7 +135,7 @@ case class CometWriteFilesExec(
124135
// them, while the file must carry the target table's column names.
125136
dataColumnNames = description.dataColumns.map(_.name),
126137
childSchema = CometUtils.fromAttributes(child.output),
127-
numPartitions = childRDD.getNumPartitions,
138+
numPartitions = writeRDD.getNumPartitions,
128139
nativeMetrics = CometMetricNode.fromCometPlan(this),
129140
nodeName = nodeName)
130141

@@ -133,7 +144,7 @@ case class CometWriteFilesExec(
133144
s"Expected ${taskWrite.dataColumnNames.length} data columns to write but the child " +
134145
s"produces ${child.output.length}")
135146

136-
childRDD.mapPartitionsInternal { batches =>
147+
writeRDD.mapPartitionsInternal { batches =>
137148
CometWriteFilesExec.executeTask(description, committer, jobTrackerID, taskWrite, batches)
138149
}
139150
}

‎spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala‎

Lines changed: 181 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.apache.comet.parquet
2121

22-
import java.io.File
22+
import java.io.{File, IOException}
2323

2424
import scala.jdk.CollectionConverters._
2525
import scala.util.{Random, Using}
@@ -28,14 +28,16 @@ import org.scalactic.source.Position
2828
import org.scalatest.Tag
2929

3030
import org.apache.hadoop.fs.{FileSystem, Path}
31+
import org.apache.hadoop.mapreduce.TaskAttemptContext
3132
import org.apache.parquet.hadoop.ParquetFileReader
3233
import org.apache.parquet.hadoop.metadata.CompressionCodecName
3334
import org.apache.parquet.hadoop.util.HadoopInputFile
35+
import org.apache.spark.internal.io.FileCommitProtocol
3436
import org.apache.spark.sql.{AnalysisException, CometTestBase, DataFrame, Row, SaveMode}
3537
import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec, CometScanExec, CometWriteFilesExec}
3638
import org.apache.spark.sql.execution.{FileSourceScanExec, QueryExecution, SparkPlan}
3739
import org.apache.spark.sql.execution.command.DataWritingCommandExec
38-
import org.apache.spark.sql.execution.datasources.WriteFilesExec
40+
import org.apache.spark.sql.execution.datasources.{SQLHadoopMapReduceCommitProtocol, WriteFilesExec}
3941
import org.apache.spark.sql.internal.SQLConf
4042
import org.apache.spark.sql.types.StructType
4143

@@ -736,6 +738,141 @@ class CometParquetWriterSuite extends CometTestBase {
736738
}
737739
}
738740

741+
test("dynamic partition overwrite falls back to Spark") {
742+
// A dynamic overwrite is a partitioned write, which CometWriteFiles declines - but the
743+
// consequence of getting it wrong is silent data loss across untouched partitions, so assert
744+
// the fallback and the semantics explicitly rather than relying on the partitioning check.
745+
withTempPath { dir =>
746+
val outputPath = new File(dir, "output.parquet").getAbsolutePath
747+
val original = Seq((1, "a"), (2, "b")).toDF("id", "part")
748+
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
749+
original.write.partitionBy("part").parquet(outputPath)
750+
}
751+
752+
withNativeWriter {
753+
withSQLConf(SQLConf.PARTITION_OVERWRITE_MODE.key -> "DYNAMIC") {
754+
val replacement = Seq((3, "b")).toDF("id", "part")
755+
val plan = captureWritePlan(
756+
p => replacement.write.mode(SaveMode.Overwrite).partitionBy("part").parquet(p),
757+
outputPath)
758+
assertNoCometWriteFilesExec(plan)
759+
}
760+
}
761+
762+
// part=a is untouched, part=b is replaced: the defining property of a dynamic overwrite.
763+
checkAnswer(spark.read.parquet(outputPath), Row(1, "a") :: Row(3, "b") :: Nil)
764+
}
765+
}
766+
767+
test("write with maxRecordsPerFile falls back to Spark") {
768+
// Spark's SingleDirectoryDataWriter rolls a new file every maxRecordsPerFile rows, bumping the
769+
// -c<counter> suffix. Comet asks the commit protocol for one file per task, so it must decline
770+
// rather than quietly produce a different file layout.
771+
Seq(
772+
"spark.sql.files.maxRecordsPerFile" -> ((df: DataFrame, p: String) => df.write.parquet(p)),
773+
// The write option takes precedence over the conf in FileFormatWriter, so it must be
774+
// honored here too - with the conf left at its default of 0.
775+
"maxRecordsPerFile-option" -> ((df: DataFrame, p: String) =>
776+
df.write.option("maxRecordsPerFile", "10").parquet(p))).foreach { case (label, write) =>
777+
withTempPath { dir =>
778+
val outputPath = new File(dir, "output.parquet").getAbsolutePath
779+
val sourcePath = new File(dir, "source.parquet").getAbsolutePath
780+
withNativeWriter {
781+
val df = materializeAsCometSource(
782+
(1 to 100).map(i => (i, s"str_$i")).toDF("id", "name").repartition(1),
783+
sourcePath)
784+
val confs =
785+
if (label == "spark.sql.files.maxRecordsPerFile") Seq(label -> "10") else Seq.empty
786+
withSQLConf(confs: _*) {
787+
val plan = captureWritePlan(p => write(df, p), outputPath)
788+
assertNoCometWriteFilesExec(plan)
789+
}
790+
checkAnswer(spark.read.parquet(outputPath), df)
791+
}
792+
// Spark's writer rolled the 100 rows of the single partition into 10 files of 10 rows.
793+
assert(
794+
listPartFileNames(outputPath).size == 10,
795+
s"$label: expected 10 rolled part files, got ${listPartFileNames(outputPath)}")
796+
}
797+
}
798+
}
799+
800+
test("empty input still writes a schema-only file (SPARK-23271)") {
801+
// An empty input must still leave a schema behind for downstream readers: `spark.read.parquet`
802+
// of the output must see the write's schema, not fail. Comet reaches this in two ways - if the
803+
// native child has one partition producing no batches, the partition-0 branch of executeTask
804+
// writes a metadata-only file; if it produces zero partitions, doExecuteWrite swaps in a dummy
805+
// single-partition RDD to get to the same branch. This test exercises the reachable path
806+
// (filtered Comet scan yielding an empty batch iterator); the zero-partition swap is defensive
807+
// because CometWriteFiles.requiresNativeChildren rules out the sources (LocalTableScan) that
808+
// would otherwise produce a zero-partition RDD.
809+
withTempPath { dir =>
810+
val outputPath = new File(dir, "output.parquet").getAbsolutePath
811+
val sourcePath = new File(dir, "source.parquet").getAbsolutePath
812+
withNativeWriter {
813+
val empty = materializeAsCometSource(
814+
(1 to 100).map(i => (i, s"str_$i")).toDF("id", "name"),
815+
sourcePath).where("id < 0")
816+
val plan = captureWritePlan(p => empty.write.parquet(p), outputPath)
817+
assertHasCometWriteFilesExec(plan)
818+
819+
val partFiles = listPartFileNames(outputPath)
820+
assert(partFiles.size == 1, s"Expected exactly one schema-only part file, got $partFiles")
821+
// Reading without an explicit schema is the point: the file must carry it.
822+
val readBack = spark.read.parquet(outputPath)
823+
assert(readBack.count() == 0L)
824+
assert(readBack.schema.map(_.name) == Seq("id", "name"))
825+
}
826+
}
827+
}
828+
829+
test("a failing task aborts, cleans up its staging file, and the retry succeeds") {
830+
// CometWriteFilesExec.executeTask must call committer.abortTask and rethrow. Injecting the
831+
// failure through the commit protocol rather than the data lets the write get as far as
832+
// creating a staging file, so the cleanup is actually observable.
833+
withTempPath { dir =>
834+
val outputPath = new File(dir, "output.parquet").getAbsolutePath
835+
val sourcePath = new File(dir, "source.parquet").getAbsolutePath
836+
withNativeWriter {
837+
val df = materializeAsCometSource(
838+
(1 to 100).map(i => (i, s"str_$i")).toDF("id", "name").repartition(1),
839+
sourcePath)
840+
841+
FailingCommitProtocol.reset()
842+
try {
843+
withSQLConf(
844+
SQLConf.FILE_COMMIT_PROTOCOL_CLASS.key -> classOf[FailingCommitProtocol].getName) {
845+
FailingCommitProtocol.failOnCommitTask = true
846+
val e = intercept[Exception] {
847+
df.write.parquet(outputPath)
848+
}
849+
assert(
850+
causeChain(e).exists(_.getMessage == FailingCommitProtocol.message),
851+
s"Expected the injected failure to propagate, got: $e")
852+
assert(
853+
FailingCommitProtocol.abortTaskCalled,
854+
"CometWriteFilesExec must call committer.abortTask when the task fails")
855+
}
856+
} finally {
857+
FailingCommitProtocol.reset()
858+
}
859+
860+
// The failed job leaves no data behind: no committed part files and no staging tree.
861+
assert(
862+
listPartFileNames(outputPath).isEmpty,
863+
s"A failed write must not leave part files: ${listPartFileNames(outputPath)}")
864+
assert(
865+
!new File(outputPath, "_temporary").exists(),
866+
"A failed write must not leave a _temporary staging directory behind")
867+
868+
// The same write, without the injected failure, still produces correct output.
869+
val plan = captureWritePlan(p => df.write.mode(SaveMode.Overwrite).parquet(p), outputPath)
870+
assertHasCometWriteFilesExec(plan)
871+
checkAnswer(spark.read.parquet(outputPath), df)
872+
}
873+
}
874+
}
875+
739876
private def createTestData(inputDir: File): String = {
740877
val inputPath = new File(inputDir, "input.parquet").getAbsolutePath
741878
val schema = FuzzDataGenerator.generateSchema(
@@ -771,6 +908,10 @@ class CometParquetWriterSuite extends CometTestBase {
771908
spark.read.parquet(sourcePath)
772909
}
773910

911+
/** `t` and everything it wraps: a task failure reaches the driver inside a SparkException. */
912+
private def causeChain(t: Throwable): Seq[Throwable] =
913+
Iterator.iterate(t)(_.getCause).takeWhile(_ != null).toSeq
914+
774915
private def listPartFileNames(dir: String): Set[String] = {
775916
val outputDir = new File(dir)
776917
if (!outputDir.exists() || !outputDir.isDirectory) {
@@ -1036,3 +1177,41 @@ class CometParquetWriterSuite extends CometTestBase {
10361177
}
10371178

10381179
}
1180+
1181+
/**
1182+
* A commit protocol that fails `commitTask` on demand, to exercise the abort branch of
1183+
* `CometWriteFilesExec.executeTask`.
1184+
*
1185+
* Failing at commit rather than mid-write means the task has already asked for a staging file and
1186+
* written it, so `abortTask` has something real to clean up. Spark instantiates this reflectively
1187+
* per job via `spark.sql.sources.commitProtocolClass`, hence the companion object for the flags -
1188+
* the tests run in `local[*]`, so executors share the driver's JVM and see them.
1189+
*/
1190+
class FailingCommitProtocol(jobId: String, path: String, dynamicPartitionOverwrite: Boolean)
1191+
extends SQLHadoopMapReduceCommitProtocol(jobId, path, dynamicPartitionOverwrite) {
1192+
1193+
override def commitTask(
1194+
taskContext: TaskAttemptContext): FileCommitProtocol.TaskCommitMessage = {
1195+
if (FailingCommitProtocol.failOnCommitTask) {
1196+
throw new IOException(FailingCommitProtocol.message)
1197+
}
1198+
super.commitTask(taskContext)
1199+
}
1200+
1201+
override def abortTask(taskContext: TaskAttemptContext): Unit = {
1202+
FailingCommitProtocol.abortTaskCalled = true
1203+
super.abortTask(taskContext)
1204+
}
1205+
}
1206+
1207+
object FailingCommitProtocol {
1208+
val message = "injected commitTask failure"
1209+
1210+
@volatile var failOnCommitTask: Boolean = false
1211+
@volatile var abortTaskCalled: Boolean = false
1212+
1213+
def reset(): Unit = {
1214+
failOnCommitTask = false
1215+
abortTaskCalled = false
1216+
}
1217+
}

0 commit comments

Comments
 (0)