support MapType

kiszk · kiszk · commit 6271839aa68e · 2016-11-15T02:49:44.000+09:00
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
@@ -94,5 +94,10 @@ object MapType extends AbstractDataType {
    * The `valueContainsNull` is true.
    */
   def apply(keyType: DataType, valueType: DataType): MapType =
-    MapType(keyType: DataType, valueType: DataType, valueContainsNull = true)
+    MapType(keyType: DataType, valueType: DataType, valueContainsNull =
+      valueType match {
+        case BooleanType | ByteType | ShortType | IntegerType | LongType |
+              FloatType | DoubleType => false
+        case _ => true
+      })
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -957,10 +957,18 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
     assert(df6.schema(1).nullable == false)
     assert(df6.schema(2).nullable == false)
 
-    val df7 = Seq(TestDataPoint(1, 2.2, "a"), TestDataPoint(3, 4.4, "null")).toDF
-    assert(df7.schema(0).nullable == false)
-    assert(df7.schema(1).nullable == false)
-    assert(df7.schema(2).nullable == true)
+    val df7 = (Tuple1(Array(1, 2, 3)) :: Nil).toDF("a")
+    assert(df7.schema(0).nullable == true)
+    assert(df7.schema(0).dataType.asInstanceOf[ArrayType].containsNull == false)
+
+    val df8 = (Tuple1(Map(2 -> 3)) :: Nil).toDF("m")
+    assert(df8.schema(0).nullable == true)
+    assert(df8.schema(0).dataType.asInstanceOf[MapType].valueContainsNull == false)
+
+    val df9 = Seq(TestDataPoint(1, 2.2, "a"), TestDataPoint(3, 4.4, "null")).toDF
+    assert(df9.schema(0).nullable == false)
+    assert(df9.schema(1).nullable == false)
+    assert(df9.schema(2).nullable == true)
   }
 
   Seq(true, false).foreach { eager =>
