Java to python

ahirreddy · ahirreddy · commit bcc0f23deb3f · 2014-04-14T17:05:18.000-07:00
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -25,7 +25,7 @@ import java.util.{List => JList, ArrayList => JArrayList, Map => JMap, Collectio
 import scala.collection.JavaConversions._
 import scala.reflect.ClassTag
 
-import net.razorvine.pickle.Unpickler
+import net.razorvine.pickle.{Pickler, Unpickler}
 
 import org.apache.spark._
 import org.apache.spark.api.java.{JavaSparkContext, JavaPairRDD, JavaRDD}
@@ -294,6 +294,15 @@ private[spark] object PythonRDD {
       }
     }
   }
+
+  def javaToPython(jRDD: JavaRDD[Any]): JavaRDD[Array[Byte]] = {
+    jRDD.rdd.mapPartitions { iter =>
+      val unpickle = new Pickler
+      iter.map { row =>
+        unpickle.dumps(row)
+      }
+    }
+  }
 }
 
 private

Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@ import java.util.{List => JList, ArrayList => JArrayList, Map => JMap, Collectio`
`25`	`25`	`import scala.collection.JavaConversions._`
`26`	`26`	`import scala.reflect.ClassTag`
`27`	`27`
`28`		`-import net.razorvine.pickle.Unpickler`
	`28`	`+import net.razorvine.pickle.{Pickler, Unpickler}`
`29`	`29`
`30`	`30`	`import org.apache.spark._`
`31`	`31`	`import org.apache.spark.api.java.{JavaSparkContext, JavaPairRDD, JavaRDD}`
`@@ -294,6 +294,15 @@ private[spark] object PythonRDD {`
`294`	`294`	`}`
`295`	`295`	`}`
`296`	`296`	`}`
	`297`	`+`
	`298`	`+ def javaToPython(jRDD: JavaRDD[Any]): JavaRDD[Array[Byte]] = {`
	`299`	`+ jRDD.rdd.mapPartitions { iter =>`
	`300`	`+ val unpickle = new Pickler`
	`301`	`+ iter.map { row =>`
	`302`	`+ unpickle.dumps(row)`
	`303`	`+ }`
	`304`	`+ }`
	`305`	`+ }`
`297`	`306`	`}`
`298`	`307`
`299`	`308`	`private`