Skip to content

Commit 8896507

Browse files
committed
handle empty string
1 parent 7b85c51 commit 8896507

3 files changed

Lines changed: 13 additions & 8 deletions

File tree

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ object ExternalCatalogUtils {
118118
}
119119

120120
def getPartitionPathString(col: String, value: String): String = {
121-
val partitionString = if (value == null) {
121+
val partitionString = if (value == null || value.isEmpty) {
122122
DEFAULT_PARTITION_NAME
123123
} else {
124124
escapePathName(value)

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -335,14 +335,11 @@ object FileFormatWriter extends Logging {
335335
/** Expressions that given partition columns build a path string like: col1=val/col2=val/... */
336336
private def partitionPathExpression: Seq[Expression] = {
337337
desc.partitionColumns.zipWithIndex.flatMap { case (c, i) =>
338-
val escaped = ScalaUDF(
339-
ExternalCatalogUtils.escapePathName _,
338+
val partitionName = ScalaUDF(
339+
ExternalCatalogUtils.getPartitionPathString _,
340340
StringType,
341-
Seq(Cast(c, StringType, Option(desc.timeZoneId))),
342-
Seq(StringType))
343-
val str = If(IsNull(c), Literal(ExternalCatalogUtils.DEFAULT_PARTITION_NAME), escaped)
344-
val partitionName = Literal(ExternalCatalogUtils.escapePathName(c.name) + "=") :: str :: Nil
345-
if (i == 0) partitionName else Literal(Path.SEPARATOR) :: partitionName
341+
Seq(Literal(c.name), Cast(c, StringType, Option(desc.timeZoneId))))
342+
if (i == 0) Seq(partitionName) else Seq(Literal(Path.SEPARATOR), partitionName)
346343
}
347344
}
348345

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,14 @@ class PartitionProviderCompatibilitySuite
328328
.write.mode("append").partitionBy("b", "c").saveAsTable("test")
329329
checkAnswer(spark.table("test"),
330330
Row(1, "p", 1) :: Row(2, null, 2) :: Row(3, null, 3) :: Nil)
331+
// make sure partition pruning also works.
332+
checkAnswer(spark.table("test").filter($"b".isNotNull), Row(1, "p", 1))
333+
334+
// empty string is an invalid partition value and we treat it as null when read back.
335+
Seq((4, "", 4)).toDF("a", "b", "c")
336+
.write.mode("append").partitionBy("b", "c").saveAsTable("test")
337+
checkAnswer(spark.table("test"),
338+
Row(1, "p", 1) :: Row(2, null, 2) :: Row(3, null, 3) :: Row(4, null, 4) :: Nil)
331339
}
332340
}
333341
}

0 commit comments

Comments
 (0)