initial commit

kiszk · kiszk · commit ac86eedfc36c · 2017-07-08T18:15:32.000+09:00
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
@@ -70,7 +70,9 @@ object TypeUtils {
 
   def compareBinary(x: Array[Byte], y: Array[Byte]): Int = {
     for (i <- 0 until x.length; if i < y.length) {
-      val res = x(i).compareTo(y(i))
+      val v1 = x(i) & 0xff
+      val v2 = y(i) & 0xff
+      val res = v1 - v2
       if (res != 0) return res
     }
     x.length - y.length
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala
@@ -137,4 +137,22 @@ class OrderingSuite extends SparkFunSuite with ExpressionEvalHelper {
     // verify that we can support up to 5000 ordering comparisons, which should be sufficient
     GenerateOrdering.generate(Array.fill(5000)(sortOrder))
   }
+
+  test("SPARK-21344: BinaryType comparison does signed byte array comparison") {
+    val b1 = Array[Byte](1)   // 0x01
+    val b2 = Array[Byte](-1)  // 0xff
+
+    val rowOrdering = InterpretedOrdering.forSchema(Seq(BinaryType, BinaryType))
+    val genOrdering = GenerateOrdering.generate(
+      BoundReference(0, BinaryType, nullable = true).asc ::
+        BoundReference(1, BinaryType, nullable = true).asc :: Nil)
+    val rowType = StructType(
+      StructField("b1", BinaryType, nullable = true) ::
+        StructField("b2", BinaryType, nullable = true) :: Nil)
+    val toCatalyst = CatalystTypeConverters.createToCatalystConverter(rowType)
+    val rowB1 = toCatalyst(Row(b1)).asInstanceOf[InternalRow]
+    val rowB2 = toCatalyst(Row(b2)).asInstanceOf[InternalRow]
+    assert(rowOrdering.compare(rowB1, rowB2) < 0)
+    assert(genOrdering.compare(rowB1, rowB2) < 0)
+  }
 }
