Skip to content

Commit 3806494

Browse files
committed
Update Arguments of ArrowUtils#toArrowSchema Function
1 parent 8f61835 commit 3806494

2 files changed

Lines changed: 23 additions & 4 deletions

File tree

‎externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import org.apache.spark.sql.types._
2828
import org.apache.kyuubi.{KyuubiSQLException, Logging}
2929
import org.apache.kyuubi.config.KyuubiConf.OPERATION_RESULT_MAX_ROWS
3030
import org.apache.kyuubi.engine.spark.KyuubiSparkUtil._
31+
import org.apache.kyuubi.engine.spark.session.SparkSessionImpl
3132
import org.apache.kyuubi.operation.{ArrayFetchIterator, FetchIterator, IterableFetchIterator, OperationHandle, OperationState}
3233
import org.apache.kyuubi.operation.log.OperationLog
3334
import org.apache.kyuubi.session.Session
@@ -185,6 +186,8 @@ class ArrowBasedExecuteStatement(
185186
incrementalCollect,
186187
handle) {
187188

189+
checkUseLargeVarType()
190+
188191
override protected def incrementalCollectResult(resultDF: DataFrame): Iterator[Any] = {
189192
toArrowBatchLocalIterator(convertComplexType(resultDF))
190193
}
@@ -204,4 +207,17 @@ class ArrowBasedExecuteStatement(
204207
private def convertComplexType(df: DataFrame): DataFrame = {
205208
convertTopLevelComplexTypeToHiveString(df, timestampAsString)
206209
}
210+
211+
def checkUseLargeVarType(): Unit = {
212+
// TODO: largeVarType support, see SPARK-39979.
213+
val useLargeVarType = session.asInstanceOf[SparkSessionImpl].spark
214+
.conf
215+
.get("spark.sql.execution.arrow.useLargeVarType", "false")
216+
.toBoolean
217+
if (useLargeVarType) {
218+
throw new KyuubiSQLException(
219+
"`spark.sql.execution.arrow.useLargeVarType = true` not support now.",
220+
null)
221+
}
222+
}
207223
}

‎externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging {
6464
"slice",
6565
0,
6666
Long.MaxValue)
67-
val arrowSchema = toArrowSchema(schema, timeZoneId, true)
67+
val arrowSchema = toArrowSchema(schema, timeZoneId, true, false)
6868
vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, sliceAllocator)
6969
try {
7070
val recordBatch = MessageSerializer.deserializeRecordBatch(
@@ -242,7 +242,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging {
242242
context: TaskContext)
243243
extends Iterator[Array[Byte]] {
244244

245-
protected val arrowSchema = toArrowSchema(schema, timeZoneId, true)
245+
protected val arrowSchema = toArrowSchema(schema, timeZoneId, true, false)
246246
private val allocator =
247247
ArrowUtils.rootAllocator.newChildAllocator(
248248
s"to${this.getClass.getSimpleName}",
@@ -327,6 +327,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging {
327327
"org.apache.spark.sql.util.ArrowUtils",
328328
classOf[StructType],
329329
classOf[String],
330+
classOf[Boolean],
330331
classOf[Boolean])
331332
.build()
332333

@@ -336,12 +337,14 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging {
336337
private def toArrowSchema(
337338
schema: StructType,
338339
timeZone: String,
339-
errorOnDuplicatedFieldNames: JBoolean): ArrowSchema = {
340+
errorOnDuplicatedFieldNames: JBoolean,
341+
largeVarTypes: JBoolean): ArrowSchema = {
340342
toArrowSchemaMethod.invoke[ArrowSchema](
341343
ArrowUtils,
342344
schema,
343345
timeZone,
344-
errorOnDuplicatedFieldNames)
346+
errorOnDuplicatedFieldNames,
347+
largeVarTypes)
345348
}
346349

347350
// IpcOption.DEFAULT was introduced in ARROW-11081(ARROW-4.0.0), add this for adapt Spark-3.1/3.2

0 commit comments

Comments
 (0)