Skip to content

Commit 81f1166

Browse files
committed
test: compare what a failed job commit leaves pending with Spark's writer
S3A's magic committer leaves the uploads of committed tasks pending when job commit fails, for Spark's own writer as well, so check that the native write leaves the same and nothing visible. Scope each pending upload check to its table, and abort what a test leaves behind.
1 parent 50efcb6 commit 81f1166

1 file changed

Lines changed: 97 additions & 40 deletions

File tree

‎spark/src/test/scala/org/apache/comet/parquet/ParquetWriteWithMagicCommitterSuite.scala‎

Lines changed: 97 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import org.apache.spark.sql.internal.SQLConf
3838
import org.apache.comet.CometS3TestBase
3939
import org.apache.comet.serde.operator.NativeWriteUtils
4040

41-
import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest
41+
import software.amazon.awssdk.services.s3.model.{AbortMultipartUploadRequest, ListMultipartUploadsRequest}
4242

4343
/**
4444
* Native Parquet writes to S3 that S3A's magic committer commits, against MinIO.
@@ -200,11 +200,35 @@ class ParquetWriteWithMagicCommitterSuite
200200
}
201201
}
202202

203-
/**
204-
* Checks that a failed write to `table` left nothing behind: no object under it, so no data and
205-
* no `__magic` directory, and no pending upload in its bucket.
206-
*/
207-
private def assertRolledBack(table: String): Unit = {
203+
/** The multipart uploads still pending under `table`, as their object keys. */
204+
private def pendingUploadsUnder(table: String): Seq[String] = {
205+
val uri = new Path(table).toUri
206+
val prefix = uri.getPath.stripPrefix("/") + "/"
207+
pendingUploads(uri.getHost).filter(_.startsWith(prefix))
208+
}
209+
210+
/** Aborts the multipart uploads still pending under `table`, so later tests start clean. */
211+
private def abortPendingUploadsUnder(table: String): Unit = withS3Client { client =>
212+
val uri = new Path(table).toUri
213+
val prefix = uri.getPath.stripPrefix("/") + "/"
214+
client
215+
.listMultipartUploads(
216+
ListMultipartUploadsRequest.builder().bucket(uri.getHost).prefix(prefix).build())
217+
.uploads()
218+
.asScala
219+
.foreach { upload =>
220+
client.abortMultipartUpload(
221+
AbortMultipartUploadRequest
222+
.builder()
223+
.bucket(uri.getHost)
224+
.key(upload.key())
225+
.uploadId(upload.uploadId())
226+
.build())
227+
}
228+
}
229+
230+
/** Checks that a failed write left no object under `table`, so no data and no `__magic`. */
231+
private def assertNothingVisible(table: String): Unit = {
208232
val tablePath = new Path(table)
209233
val fs = tablePath.getFileSystem(spark.sessionState.newHadoopConf())
210234
if (fs.exists(tablePath)) {
@@ -213,8 +237,12 @@ class ParquetWriteWithMagicCommitterSuite
213237
Iterator.continually(iterator).takeWhile(_.hasNext).map(_.next().getPath).toList
214238
assert(files.isEmpty, s"A failed write left ${files.mkString(", ")}")
215239
}
216-
val bucket = tablePath.toUri.getHost
217-
assert(pendingUploads(bucket).isEmpty, pendingUploads(bucket))
240+
}
241+
242+
/** Checks that a failed write to `table` left nothing behind, not even a pending upload. */
243+
private def assertRolledBack(table: String): Unit = {
244+
assertNothingVisible(table)
245+
assert(pendingUploadsUnder(table).isEmpty, pendingUploadsUnder(table))
218246
}
219247

220248
test("a native write that fails after S3A uploaded part of its file leaves nothing behind") {
@@ -252,40 +280,69 @@ class ParquetWriteWithMagicCommitterSuite
252280
}
253281
}
254282

255-
// A commit that fails after the file is closed. S3A has recorded the file's pending upload by
256-
// then, so only the magic committer's own abort can cancel it: abortTask for a failed task
257-
// commit, and abortJob for a failed job commit, which on Spark 3.x is Comet's own call.
258-
Seq(
259-
"task" -> (() => FailingPathOutputCommitProtocol.failCommitTask = true,
260-
FailingPathOutputCommitProtocol.CommitTaskFailure),
261-
"job" -> (() => FailingPathOutputCommitProtocol.failCommitJob = true,
262-
FailingPathOutputCommitProtocol.CommitJobFailure)).foreach {
263-
case (stage, (failCommit, failure)) =>
264-
test(s"a native write whose $stage commit fails leaves nothing behind") {
265-
withTempPath { dir =>
266-
val df = source(1000, dir).coalesce(1)
267-
val table = s"s3a://$testBucketName/failed-$stage-commit"
268-
FailingPathOutputCommitProtocol.reset()
269-
try {
270-
withNativeWriter {
271-
withSQLConf(
272-
SQLConf.FILE_COMMIT_PROTOCOL_CLASS.key ->
273-
classOf[FailingPathOutputCommitProtocol].getName,
274-
"fs.s3a.committer.abort.pending.uploads" -> "false") {
275-
failCommit()
276-
val (plan, error) = captureFailedWritePlan(df.write.parquet(table))
277-
assertHasCometNativeWriteExec(plan)
278-
assert(
279-
causeChain(error).exists(e => Option(e.getMessage).exists(_.contains(failure))),
280-
s"Expected the injected $stage commit failure, got: $error")
281-
}
282-
}
283-
} finally {
284-
FailingPathOutputCommitProtocol.reset()
285-
}
286-
assertRolledBack(table)
283+
/**
284+
* Writes 1000 rows in one task to `table` under FailingPathOutputCommitProtocol, which `fail`
285+
* sets up to fail a commit, natively or with Spark's writer, and checks that the write failed
286+
* with `failure` and left no object behind. Returns the uploads it left pending.
287+
*/
288+
private def failCommit(table: String, native: Boolean, failure: String)(
289+
fail: => Unit): Seq[String] = {
290+
var pending = Seq.empty[String]
291+
withTempPath { dir =>
292+
val df = source(1000, dir).coalesce(1)
293+
FailingPathOutputCommitProtocol.reset()
294+
try {
295+
def write(): Unit = withSQLConf(
296+
SQLConf.FILE_COMMIT_PROTOCOL_CLASS.key ->
297+
classOf[FailingPathOutputCommitProtocol].getName,
298+
"fs.s3a.committer.abort.pending.uploads" -> "false") {
299+
fail
300+
val (plan, error) = captureFailedWritePlan(df.write.parquet(table))
301+
if (native) assertHasCometNativeWriteExec(plan) else assertNoCometNativeWriteExec(plan)
302+
assert(
303+
causeChain(error).exists(e => Option(e.getMessage).exists(_.contains(failure))),
304+
s"Expected the injected commit failure, got: $error")
287305
}
306+
if (native) withNativeWriter(write()) else write()
307+
} finally {
308+
FailingPathOutputCommitProtocol.reset()
288309
}
310+
assertNothingVisible(table)
311+
try pending = pendingUploadsUnder(table)
312+
finally abortPendingUploadsUnder(table)
313+
}
314+
pending
315+
}
316+
317+
test("a native write whose task commit fails leaves nothing behind") {
318+
// The file is closed before the task commits, so S3A has recorded its pending upload, and only
319+
// the magic committer's abortTask can cancel it.
320+
val pending = failCommit(
321+
s"s3a://$testBucketName/failed-task-commit",
322+
native = true,
323+
FailingPathOutputCommitProtocol.CommitTaskFailure) {
324+
FailingPathOutputCommitProtocol.failCommitTask = true
325+
}
326+
assert(pending.isEmpty, pending)
327+
}
328+
329+
test("a native write whose job commit fails leaves what Spark's writer leaves") {
330+
// The magic committer's abortJob, which on Spark 3.x is Comet's own call, removes `__magic`
331+
// but leaves the uploads of the tasks that committed pending, for Spark's own writer too and
332+
// whether or not fs.s3a.committer.abort.pending.uploads is set. So the native write has to
333+
// leave nothing visible and exactly what Spark's writer leaves pending.
334+
def failJobCommit(native: Boolean): Seq[String] =
335+
failCommit(
336+
s"s3a://$testBucketName/failed-job-commit-native-$native",
337+
native,
338+
FailingPathOutputCommitProtocol.CommitJobFailure) {
339+
FailingPathOutputCommitProtocol.failCommitJob = true
340+
}
341+
val leftBySpark = failJobCommit(native = false)
342+
val leftByComet = failJobCommit(native = true)
343+
assert(
344+
leftByComet.size == leftBySpark.size,
345+
s"Spark's writer left $leftBySpark pending, the native writer $leftByComet")
289346
}
290347

291348
test("an S3 write that S3A's magic committer does not commit stays on Spark") {

0 commit comments

Comments
 (0)