Skip to content

Commit be0533a

Browse files
committed
Address the comments
1 parent a82b184 commit be0533a

File tree

1 file changed

+6
-3
lines changed

1 file changed

+6
-3
lines changed

core/src/main/scala/org/apache/spark/util/Utils.scala

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -269,11 +269,14 @@ private[spark] object Utils extends Logging {
269269
dir
270270
}
271271

272-
/** Copy all data from an InputStream to an OutputStream */
272+
/** Copy all data from an InputStream to an OutputStream. NIO way of file stream to file stream
273+
* copying is disabled by default unless explicitly set transferToEnabled as true,
274+
* the parameter transferToEnabled should be configured by spark.file.transferTo = [true|false].
275+
*/
273276
def copyStream(in: InputStream,
274277
out: OutputStream,
275278
closeStreams: Boolean = false,
276-
transferToEnabled: Boolean = true): Long =
279+
transferToEnabled: Boolean = false): Long =
277280
{
278281
var count = 0L
279282
try {
@@ -299,7 +302,7 @@ private[spark] object Utils extends Logging {
299302
val finalPos = outChannel.position()
300303
assert(finalPos == initialPos + size,
301304
s"""
302-
|Current position $finalPos do not equal to expected position ${initialPos + count}
305+
|Current position $finalPos do not equal to expected position ${initialPos + size}
303306
|after transferTo, please check your kernel version to see if it is 2.6.32,
304307
|this is a kernel bug which will lead to unexpected behavior when using transferTo.
305308
|You can set spark.file.transferTo = false to disable this NIO feature.

0 commit comments

Comments
 (0)