@@ -38,7 +38,7 @@ import org.apache.spark.sql.internal.SQLConf
3838import org .apache .comet .CometS3TestBase
3939import org .apache .comet .serde .operator .NativeWriteUtils
4040
41- import software .amazon .awssdk .services .s3 .model .ListMultipartUploadsRequest
41+ import software .amazon .awssdk .services .s3 .model .{ AbortMultipartUploadRequest , ListMultipartUploadsRequest }
4242
4343/**
4444 * Native Parquet writes to S3 that S3A's magic committer commits, against MinIO.
@@ -200,11 +200,35 @@ class ParquetWriteWithMagicCommitterSuite
200200 }
201201 }
202202
203- /**
204- * Checks that a failed write to `table` left nothing behind: no object under it, so no data and
205- * no `__magic` directory, and no pending upload in its bucket.
206- */
207- private def assertRolledBack (table : String ): Unit = {
203+ /** The multipart uploads still pending under `table`, as their object keys. */
204+ private def pendingUploadsUnder (table : String ): Seq [String ] = {
205+ val uri = new Path (table).toUri
206+ val prefix = uri.getPath.stripPrefix(" /" ) + " /"
207+ pendingUploads(uri.getHost).filter(_.startsWith(prefix))
208+ }
209+
210+ /** Aborts the multipart uploads still pending under `table`, so later tests start clean. */
211+ private def abortPendingUploadsUnder (table : String ): Unit = withS3Client { client =>
212+ val uri = new Path (table).toUri
213+ val prefix = uri.getPath.stripPrefix(" /" ) + " /"
214+ client
215+ .listMultipartUploads(
216+ ListMultipartUploadsRequest .builder().bucket(uri.getHost).prefix(prefix).build())
217+ .uploads()
218+ .asScala
219+ .foreach { upload =>
220+ client.abortMultipartUpload(
221+ AbortMultipartUploadRequest
222+ .builder()
223+ .bucket(uri.getHost)
224+ .key(upload.key())
225+ .uploadId(upload.uploadId())
226+ .build())
227+ }
228+ }
229+
230+ /** Checks that a failed write left no object under `table`, so no data and no `__magic`. */
231+ private def assertNothingVisible (table : String ): Unit = {
208232 val tablePath = new Path (table)
209233 val fs = tablePath.getFileSystem(spark.sessionState.newHadoopConf())
210234 if (fs.exists(tablePath)) {
@@ -213,8 +237,12 @@ class ParquetWriteWithMagicCommitterSuite
213237 Iterator .continually(iterator).takeWhile(_.hasNext).map(_.next().getPath).toList
214238 assert(files.isEmpty, s " A failed write left ${files.mkString(" , " )}" )
215239 }
216- val bucket = tablePath.toUri.getHost
217- assert(pendingUploads(bucket).isEmpty, pendingUploads(bucket))
240+ }
241+
242+ /** Checks that a failed write to `table` left nothing behind, not even a pending upload. */
243+ private def assertRolledBack (table : String ): Unit = {
244+ assertNothingVisible(table)
245+ assert(pendingUploadsUnder(table).isEmpty, pendingUploadsUnder(table))
218246 }
219247
220248 test(" a native write that fails after S3A uploaded part of its file leaves nothing behind" ) {
@@ -252,40 +280,69 @@ class ParquetWriteWithMagicCommitterSuite
252280 }
253281 }
254282
255- // A commit that fails after the file is closed. S3A has recorded the file's pending upload by
256- // then, so only the magic committer's own abort can cancel it: abortTask for a failed task
257- // commit, and abortJob for a failed job commit, which on Spark 3.x is Comet's own call.
258- Seq (
259- " task" -> (() => FailingPathOutputCommitProtocol .failCommitTask = true ,
260- FailingPathOutputCommitProtocol .CommitTaskFailure ),
261- " job" -> (() => FailingPathOutputCommitProtocol .failCommitJob = true ,
262- FailingPathOutputCommitProtocol .CommitJobFailure )).foreach {
263- case (stage, (failCommit, failure)) =>
264- test(s " a native write whose $stage commit fails leaves nothing behind " ) {
265- withTempPath { dir =>
266- val df = source(1000 , dir).coalesce(1 )
267- val table = s " s3a:// $testBucketName/failed- $stage-commit "
268- FailingPathOutputCommitProtocol .reset()
269- try {
270- withNativeWriter {
271- withSQLConf(
272- SQLConf .FILE_COMMIT_PROTOCOL_CLASS .key ->
273- classOf [FailingPathOutputCommitProtocol ].getName,
274- " fs.s3a.committer.abort.pending.uploads" -> " false" ) {
275- failCommit()
276- val (plan, error) = captureFailedWritePlan(df.write.parquet(table))
277- assertHasCometNativeWriteExec(plan)
278- assert(
279- causeChain(error).exists(e => Option (e.getMessage).exists(_.contains(failure))),
280- s " Expected the injected $stage commit failure, got: $error" )
281- }
282- }
283- } finally {
284- FailingPathOutputCommitProtocol .reset()
285- }
286- assertRolledBack(table)
283+ /**
284+ * Writes 1000 rows in one task to `table` under FailingPathOutputCommitProtocol, which `fail`
285+ * sets up to fail a commit, natively or with Spark's writer, and checks that the write failed
286+ * with `failure` and left no object behind. Returns the uploads it left pending.
287+ */
288+ private def failCommit (table : String , native : Boolean , failure : String )(
289+ fail : => Unit ): Seq [String ] = {
290+ var pending = Seq .empty[String ]
291+ withTempPath { dir =>
292+ val df = source(1000 , dir).coalesce(1 )
293+ FailingPathOutputCommitProtocol .reset()
294+ try {
295+ def write (): Unit = withSQLConf(
296+ SQLConf .FILE_COMMIT_PROTOCOL_CLASS .key ->
297+ classOf [FailingPathOutputCommitProtocol ].getName,
298+ " fs.s3a.committer.abort.pending.uploads" -> " false" ) {
299+ fail
300+ val (plan, error) = captureFailedWritePlan(df.write.parquet(table))
301+ if (native) assertHasCometNativeWriteExec(plan) else assertNoCometNativeWriteExec(plan)
302+ assert(
303+ causeChain(error).exists(e => Option (e.getMessage).exists(_.contains(failure))),
304+ s " Expected the injected commit failure, got: $error" )
287305 }
306+ if (native) withNativeWriter(write()) else write()
307+ } finally {
308+ FailingPathOutputCommitProtocol .reset()
288309 }
310+ assertNothingVisible(table)
311+ try pending = pendingUploadsUnder(table)
312+ finally abortPendingUploadsUnder(table)
313+ }
314+ pending
315+ }
316+
317+ test(" a native write whose task commit fails leaves nothing behind" ) {
318+ // The file is closed before the task commits, so S3A has recorded its pending upload, and only
319+ // the magic committer's abortTask can cancel it.
320+ val pending = failCommit(
321+ s " s3a:// $testBucketName/failed-task-commit " ,
322+ native = true ,
323+ FailingPathOutputCommitProtocol .CommitTaskFailure ) {
324+ FailingPathOutputCommitProtocol .failCommitTask = true
325+ }
326+ assert(pending.isEmpty, pending)
327+ }
328+
329+ test(" a native write whose job commit fails leaves what Spark's writer leaves" ) {
330+ // The magic committer's abortJob, which on Spark 3.x is Comet's own call, removes `__magic`
331+ // but leaves the uploads of the tasks that committed pending, for Spark's own writer too and
332+ // whether or not fs.s3a.committer.abort.pending.uploads is set. So the native write has to
333+ // leave nothing visible and exactly what Spark's writer leaves pending.
334+ def failJobCommit (native : Boolean ): Seq [String ] =
335+ failCommit(
336+ s " s3a:// $testBucketName/failed-job-commit-native- $native" ,
337+ native,
338+ FailingPathOutputCommitProtocol .CommitJobFailure ) {
339+ FailingPathOutputCommitProtocol .failCommitJob = true
340+ }
341+ val leftBySpark = failJobCommit(native = false )
342+ val leftByComet = failJobCommit(native = true )
343+ assert(
344+ leftByComet.size == leftBySpark.size,
345+ s " Spark's writer left $leftBySpark pending, the native writer $leftByComet" )
289346 }
290347
291348 test(" an S3 write that S3A's magic committer does not commit stays on Spark" ) {
0 commit comments