Skip to content

Commit 7024529

Browse files
committed
Core: Clear derived delete files when file scan tasks are cleared
BaseScanTaskResponse.Builder.withFileScanTasks derives deleteFiles from the tasks, but only on the non-null branch, so clearing the tasks left the previously derived delete files behind. Combined with the removal of withDeleteFiles in this PR, there was no longer any way to clear them, and response.toBuilder().withPlanStatus(FAILED).withFileScanTasks(null).build() failed validation with "deleteFiles should only be returned with fileScanTasks that reference them". Derives on both branches so clearing the tasks clears the derived delete files, and reads the tasks parameter rather than the field, which is what the guard always meant. Reported-by: u70b3
1 parent b6c8fcb commit 7024529

2 files changed

Lines changed: 34 additions & 5 deletions

File tree

‎core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,11 @@ public B withPlanTasks(List<String> tasks) {
8181

8282
public B withFileScanTasks(List<FileScanTask> tasks) {
8383
this.fileScanTasks = tasks;
84-
if (fileScanTasks != null) {
85-
this.deleteFiles =
86-
DeleteFileSet.of(
87-
() -> tasks.stream().flatMap(task -> task.deletes().stream()).iterator());
88-
}
84+
this.deleteFiles =
85+
tasks == null
86+
? null
87+
: DeleteFileSet.of(
88+
() -> tasks.stream().flatMap(task -> task.deletes().stream()).iterator());
8989
return self();
9090
}
9191

‎core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,35 @@ public void multipleTasksWithDifferentDeleteFilesDontAccumulateReferences() {
374374
assertThat(json).isEqualTo(expectedJson);
375375
}
376376

377+
@Test
378+
public void toBuilderClearsDeleteFilesWhenClearingFileScanTasks() {
379+
ResidualEvaluator residualEvaluator =
380+
ResidualEvaluator.of(SPEC, Expressions.alwaysTrue(), true);
381+
FileScanTask task =
382+
new BaseFileScanTask(
383+
FILE_A,
384+
new DeleteFile[] {FILE_A_DELETES},
385+
SchemaParser.toJson(SCHEMA),
386+
PartitionSpecParser.toJson(SPEC),
387+
residualEvaluator);
388+
389+
PlanTableScanResponse completed =
390+
PlanTableScanResponse.builder()
391+
.withPlanStatus(PlanStatus.COMPLETED)
392+
.withFileScanTasks(List.of(task))
393+
.withSpecsById(PARTITION_SPECS_BY_ID)
394+
.build();
395+
assertThat(completed.deleteFiles()).containsExactly(FILE_A_DELETES);
396+
397+
// clearing the tasks must clear the delete files derived from them, otherwise validate()
398+
// rejects the copy because the retained delete files reference tasks that are no longer present
399+
PlanTableScanResponse failed =
400+
completed.toBuilder().withPlanStatus(PlanStatus.FAILED).withFileScanTasks(null).build();
401+
402+
assertThat(failed.fileScanTasks()).isNull();
403+
assertThat(failed.deleteFiles()).isNull();
404+
}
405+
377406
@Test
378407
public void roundTripSerdeWithoutDeleteFiles() {
379408
ResidualEvaluator residualEvaluator =

0 commit comments

Comments
 (0)