1919
2020package org .apache .comet .parquet
2121
22- import java .io .File
22+ import java .io .{ File , IOException }
2323
2424import scala .jdk .CollectionConverters ._
2525import scala .util .{Random , Using }
@@ -28,14 +28,16 @@ import org.scalactic.source.Position
2828import org .scalatest .Tag
2929
3030import org .apache .hadoop .fs .{FileSystem , Path }
31+ import org .apache .hadoop .mapreduce .TaskAttemptContext
3132import org .apache .parquet .hadoop .ParquetFileReader
3233import org .apache .parquet .hadoop .metadata .CompressionCodecName
3334import org .apache .parquet .hadoop .util .HadoopInputFile
35+ import org .apache .spark .internal .io .FileCommitProtocol
3436import org .apache .spark .sql .{AnalysisException , CometTestBase , DataFrame , Row , SaveMode }
3537import org .apache .spark .sql .comet .{CometBatchScanExec , CometNativeScanExec , CometScanExec , CometWriteFilesExec }
3638import org .apache .spark .sql .execution .{FileSourceScanExec , QueryExecution , SparkPlan }
3739import org .apache .spark .sql .execution .command .DataWritingCommandExec
38- import org .apache .spark .sql .execution .datasources .WriteFilesExec
40+ import org .apache .spark .sql .execution .datasources .{ SQLHadoopMapReduceCommitProtocol , WriteFilesExec }
3941import org .apache .spark .sql .internal .SQLConf
4042import org .apache .spark .sql .types .StructType
4143
@@ -736,6 +738,141 @@ class CometParquetWriterSuite extends CometTestBase {
736738 }
737739 }
738740
741+ test(" dynamic partition overwrite falls back to Spark" ) {
742+ // A dynamic overwrite is a partitioned write, which CometWriteFiles declines - but the
743+ // consequence of getting it wrong is silent data loss across untouched partitions, so assert
744+ // the fallback and the semantics explicitly rather than relying on the partitioning check.
745+ withTempPath { dir =>
746+ val outputPath = new File (dir, " output.parquet" ).getAbsolutePath
747+ val original = Seq ((1 , " a" ), (2 , " b" )).toDF(" id" , " part" )
748+ withSQLConf(CometConf .COMET_ENABLED .key -> " false" ) {
749+ original.write.partitionBy(" part" ).parquet(outputPath)
750+ }
751+
752+ withNativeWriter {
753+ withSQLConf(SQLConf .PARTITION_OVERWRITE_MODE .key -> " DYNAMIC" ) {
754+ val replacement = Seq ((3 , " b" )).toDF(" id" , " part" )
755+ val plan = captureWritePlan(
756+ p => replacement.write.mode(SaveMode .Overwrite ).partitionBy(" part" ).parquet(p),
757+ outputPath)
758+ assertNoCometWriteFilesExec(plan)
759+ }
760+ }
761+
762+ // part=a is untouched, part=b is replaced: the defining property of a dynamic overwrite.
763+ checkAnswer(spark.read.parquet(outputPath), Row (1 , " a" ) :: Row (3 , " b" ) :: Nil )
764+ }
765+ }
766+
767+ test(" write with maxRecordsPerFile falls back to Spark" ) {
768+ // Spark's SingleDirectoryDataWriter rolls a new file every maxRecordsPerFile rows, bumping the
769+ // -c<counter> suffix. Comet asks the commit protocol for one file per task, so it must decline
770+ // rather than quietly produce a different file layout.
771+ Seq (
772+ " spark.sql.files.maxRecordsPerFile" -> ((df : DataFrame , p : String ) => df.write.parquet(p)),
773+ // The write option takes precedence over the conf in FileFormatWriter, so it must be
774+ // honored here too - with the conf left at its default of 0.
775+ " maxRecordsPerFile-option" -> ((df : DataFrame , p : String ) =>
776+ df.write.option(" maxRecordsPerFile" , " 10" ).parquet(p))).foreach { case (label, write) =>
777+ withTempPath { dir =>
778+ val outputPath = new File (dir, " output.parquet" ).getAbsolutePath
779+ val sourcePath = new File (dir, " source.parquet" ).getAbsolutePath
780+ withNativeWriter {
781+ val df = materializeAsCometSource(
782+ (1 to 100 ).map(i => (i, s " str_ $i" )).toDF(" id" , " name" ).repartition(1 ),
783+ sourcePath)
784+ val confs =
785+ if (label == " spark.sql.files.maxRecordsPerFile" ) Seq (label -> " 10" ) else Seq .empty
786+ withSQLConf(confs : _* ) {
787+ val plan = captureWritePlan(p => write(df, p), outputPath)
788+ assertNoCometWriteFilesExec(plan)
789+ }
790+ checkAnswer(spark.read.parquet(outputPath), df)
791+ }
792+ // Spark's writer rolled the 100 rows of the single partition into 10 files of 10 rows.
793+ assert(
794+ listPartFileNames(outputPath).size == 10 ,
795+ s " $label: expected 10 rolled part files, got ${listPartFileNames(outputPath)}" )
796+ }
797+ }
798+ }
799+
800+ test(" empty input still writes a schema-only file (SPARK-23271)" ) {
801+ // An empty input must still leave a schema behind for downstream readers: `spark.read.parquet`
802+ // of the output must see the write's schema, not fail. Comet reaches this in two ways - if the
803+ // native child has one partition producing no batches, the partition-0 branch of executeTask
804+ // writes a metadata-only file; if it produces zero partitions, doExecuteWrite swaps in a dummy
805+ // single-partition RDD to get to the same branch. This test exercises the reachable path
806+ // (filtered Comet scan yielding an empty batch iterator); the zero-partition swap is defensive
807+ // because CometWriteFiles.requiresNativeChildren rules out the sources (LocalTableScan) that
808+ // would otherwise produce a zero-partition RDD.
809+ withTempPath { dir =>
810+ val outputPath = new File (dir, " output.parquet" ).getAbsolutePath
811+ val sourcePath = new File (dir, " source.parquet" ).getAbsolutePath
812+ withNativeWriter {
813+ val empty = materializeAsCometSource(
814+ (1 to 100 ).map(i => (i, s " str_ $i" )).toDF(" id" , " name" ),
815+ sourcePath).where(" id < 0" )
816+ val plan = captureWritePlan(p => empty.write.parquet(p), outputPath)
817+ assertHasCometWriteFilesExec(plan)
818+
819+ val partFiles = listPartFileNames(outputPath)
820+ assert(partFiles.size == 1 , s " Expected exactly one schema-only part file, got $partFiles" )
821+ // Reading without an explicit schema is the point: the file must carry it.
822+ val readBack = spark.read.parquet(outputPath)
823+ assert(readBack.count() == 0L )
824+ assert(readBack.schema.map(_.name) == Seq (" id" , " name" ))
825+ }
826+ }
827+ }
828+
829+ test(" a failing task aborts, cleans up its staging file, and the retry succeeds" ) {
830+ // CometWriteFilesExec.executeTask must call committer.abortTask and rethrow. Injecting the
831+ // failure through the commit protocol rather than the data lets the write get as far as
832+ // creating a staging file, so the cleanup is actually observable.
833+ withTempPath { dir =>
834+ val outputPath = new File (dir, " output.parquet" ).getAbsolutePath
835+ val sourcePath = new File (dir, " source.parquet" ).getAbsolutePath
836+ withNativeWriter {
837+ val df = materializeAsCometSource(
838+ (1 to 100 ).map(i => (i, s " str_ $i" )).toDF(" id" , " name" ).repartition(1 ),
839+ sourcePath)
840+
841+ FailingCommitProtocol .reset()
842+ try {
843+ withSQLConf(
844+ SQLConf .FILE_COMMIT_PROTOCOL_CLASS .key -> classOf [FailingCommitProtocol ].getName) {
845+ FailingCommitProtocol .failOnCommitTask = true
846+ val e = intercept[Exception ] {
847+ df.write.parquet(outputPath)
848+ }
849+ assert(
850+ causeChain(e).exists(_.getMessage == FailingCommitProtocol .message),
851+ s " Expected the injected failure to propagate, got: $e" )
852+ assert(
853+ FailingCommitProtocol .abortTaskCalled,
854+ " CometWriteFilesExec must call committer.abortTask when the task fails" )
855+ }
856+ } finally {
857+ FailingCommitProtocol .reset()
858+ }
859+
860+ // The failed job leaves no data behind: no committed part files and no staging tree.
861+ assert(
862+ listPartFileNames(outputPath).isEmpty,
863+ s " A failed write must not leave part files: ${listPartFileNames(outputPath)}" )
864+ assert(
865+ ! new File (outputPath, " _temporary" ).exists(),
866+ " A failed write must not leave a _temporary staging directory behind" )
867+
868+ // The same write, without the injected failure, still produces correct output.
869+ val plan = captureWritePlan(p => df.write.mode(SaveMode .Overwrite ).parquet(p), outputPath)
870+ assertHasCometWriteFilesExec(plan)
871+ checkAnswer(spark.read.parquet(outputPath), df)
872+ }
873+ }
874+ }
875+
739876 private def createTestData (inputDir : File ): String = {
740877 val inputPath = new File (inputDir, " input.parquet" ).getAbsolutePath
741878 val schema = FuzzDataGenerator .generateSchema(
@@ -771,6 +908,10 @@ class CometParquetWriterSuite extends CometTestBase {
771908 spark.read.parquet(sourcePath)
772909 }
773910
911+ /** `t` and everything it wraps: a task failure reaches the driver inside a SparkException. */
912+ private def causeChain (t : Throwable ): Seq [Throwable ] =
913+ Iterator .iterate(t)(_.getCause).takeWhile(_ != null ).toSeq
914+
774915 private def listPartFileNames (dir : String ): Set [String ] = {
775916 val outputDir = new File (dir)
776917 if (! outputDir.exists() || ! outputDir.isDirectory) {
@@ -1036,3 +1177,41 @@ class CometParquetWriterSuite extends CometTestBase {
10361177 }
10371178
10381179}
1180+
1181+ /**
1182+ * A commit protocol that fails `commitTask` on demand, to exercise the abort branch of
1183+ * `CometWriteFilesExec.executeTask`.
1184+ *
1185+ * Failing at commit rather than mid-write means the task has already asked for a staging file and
1186+ * written it, so `abortTask` has something real to clean up. Spark instantiates this reflectively
1187+ * per job via `spark.sql.sources.commitProtocolClass`, hence the companion object for the flags -
1188+ * the tests run in `local[*]`, so executors share the driver's JVM and see them.
1189+ */
1190+ class FailingCommitProtocol (jobId : String , path : String , dynamicPartitionOverwrite : Boolean )
1191+ extends SQLHadoopMapReduceCommitProtocol (jobId, path, dynamicPartitionOverwrite) {
1192+
1193+ override def commitTask (
1194+ taskContext : TaskAttemptContext ): FileCommitProtocol .TaskCommitMessage = {
1195+ if (FailingCommitProtocol .failOnCommitTask) {
1196+ throw new IOException (FailingCommitProtocol .message)
1197+ }
1198+ super .commitTask(taskContext)
1199+ }
1200+
1201+ override def abortTask (taskContext : TaskAttemptContext ): Unit = {
1202+ FailingCommitProtocol .abortTaskCalled = true
1203+ super .abortTask(taskContext)
1204+ }
1205+ }
1206+
1207+ object FailingCommitProtocol {
1208+ val message = " injected commitTask failure"
1209+
1210+ @ volatile var failOnCommitTask : Boolean = false
1211+ @ volatile var abortTaskCalled : Boolean = false
1212+
1213+ def reset (): Unit = {
1214+ failOnCommitTask = false
1215+ abortTaskCalled = false
1216+ }
1217+ }
0 commit comments