Skip to content

Commit a585707

Browse files
andygroveclaude
andcommitted
fix: Close ColumnarBatch after native columnar to row conversion
Close the ColumnarBatch after the row iterator is fully consumed to ensure proper cleanup of Arrow resources. The batch is closed via a wrapper iterator that calls batch.close() when hasNext returns false. This fix applies to both the regular doExecute() path and the broadcast execution path. Co-Authored-By: Claude Opus 4.5 <[email protected]>
1 parent 44581c2 commit a585707

1 file changed

Lines changed: 23 additions & 2 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,18 @@ case class CometNativeColumnarToRowExec(child: SparkPlan)
112112
.flatMap { batch =>
113113
numInputBatches += 1
114114
numOutputRows += batch.numRows()
115-
converter.convert(batch)
115+
val result = converter.convert(batch)
116+
// Wrap iterator to close batch after consumption
117+
new Iterator[InternalRow] {
118+
override def hasNext: Boolean = {
119+
val hasMore = result.hasNext
120+
if (!hasMore) {
121+
batch.close()
122+
}
123+
hasMore
124+
}
125+
override def next(): InternalRow = result.next()
126+
}
116127
}
117128

118129
val mode = cometBroadcastExchange.get.mode
@@ -200,7 +211,17 @@ case class CometNativeColumnarToRowExec(child: SparkPlan)
200211
val result = converter.convert(batch)
201212
convertTime += System.nanoTime() - startTime
202213

203-
result
214+
// Wrap iterator to close batch after consumption
215+
new Iterator[InternalRow] {
216+
override def hasNext: Boolean = {
217+
val hasMore = result.hasNext
218+
if (!hasMore) {
219+
batch.close()
220+
}
221+
hasMore
222+
}
223+
override def next(): InternalRow = result.next()
224+
}
204225
}
205226
}
206227
}

0 commit comments

Comments
 (0)