File tree Expand file tree Collapse file tree 1 file changed +6
-3
lines changed
core/src/main/scala/org/apache/spark/util Expand file tree Collapse file tree 1 file changed +6
-3
lines changed Original file line number Diff line number Diff line change @@ -269,11 +269,14 @@ private[spark] object Utils extends Logging {
269269 dir
270270 }
271271
272- /** Copy all data from an InputStream to an OutputStream */
272+ /** Copy all data from an InputStream to an OutputStream. NIO way of file stream to file stream
273+ * copying is disabled by default unless explicitly set transferToEnabled as true,
274+ * the parameter transferToEnabled should be configured by spark.file.transferTo = [true|false].
275+ */
273276 def copyStream (in : InputStream ,
274277 out : OutputStream ,
275278 closeStreams : Boolean = false ,
276- transferToEnabled : Boolean = true ): Long =
279+ transferToEnabled : Boolean = false ): Long =
277280 {
278281 var count = 0L
279282 try {
@@ -299,7 +302,7 @@ private[spark] object Utils extends Logging {
299302 val finalPos = outChannel.position()
300303 assert(finalPos == initialPos + size,
301304 s """
302- |Current position $finalPos do not equal to expected position ${initialPos + count }
305+ |Current position $finalPos do not equal to expected position ${initialPos + size }
303306 |after transferTo, please check your kernel version to see if it is 2.6.32,
304307 |this is a kernel bug which will lead to unexpected behavior when using transferTo.
305308 |You can set spark.file.transferTo = false to disable this NIO feature.
You can’t perform that action at this time.
0 commit comments