[SPARK-34768][SQL] Respect the default input buffer size in Univocity

HyukjinKwon · HyukjinKwon · commit 385f1e8f5de5 · 2021-03-17T19:55:49.000+09:00
### What changes were proposed in this pull request? This PR proposes to follow Univocity's input buffer. ### Why are the changes needed? - Firstly, it's best to trust their judgement on the default values. Also 128 is too low. - Default values arguably have more test coverage in Univocity. - It will also fix uniVocity/univocity-parsers#449 - ^ is a regression compared to Spark 2.4 ### Does this PR introduce _any_ user-facing change? No. In addition, It fixes a regression. ### How was this patch tested? Manually tested, and added a unit test. Closes apache#31858 from HyukjinKwon/SPARK-34768. Authored-by: HyukjinKwon <gurwls223@apache.org> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -166,8 +166,6 @@ class CSVOptions(
 
   val quoteAll = getBool("quoteAll", false)
 
-  val inputBufferSize = 128
-
   /**
    * The max error content length in CSV parser/writer exception message.
    */
@@ -259,7 +257,6 @@ class CSVOptions(
     settings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceInRead)
     settings.setIgnoreTrailingWhitespaces(ignoreTrailingWhiteSpaceInRead)
     settings.setReadInputOnSeparateThread(false)
-    settings.setInputBufferSize(inputBufferSize)
     settings.setMaxColumns(maxColumns)
     settings.setNullValue(nullValue)
     settings.setEmptyValue(emptyValueInRead)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2452,6 +2452,17 @@ abstract class CSVSuite
       assert(result.sameElements(exceptResults))
     }
   }
+
+  test("SPARK-34768: counting a long record with ignoreTrailingWhiteSpace set to true") {
+    val bufSize = 128
+    val line = "X" * (bufSize - 1) + "| |"
+    withTempPath { path =>
+      Seq(line).toDF.write.text(path.getAbsolutePath)
+      assert(spark.read.format("csv")
+        .option("delimiter", "|")
+        .option("ignoreTrailingWhiteSpace", "true").load(path.getAbsolutePath).count() == 1)
+    }
+  }
 }
 
 class CSVv1Suite extends CSVSuite {

Original file line number	Diff line number	Diff line change
`@@ -2452,6 +2452,17 @@ abstract class CSVSuite`
`2452`	`2452`	`assert(result.sameElements(exceptResults))`
`2453`	`2453`	`}`
`2454`	`2454`	`}`
	`2455`	`+`
	`2456`	`+ test("SPARK-34768: counting a long record with ignoreTrailingWhiteSpace set to true") {`
	`2457`	`+ val bufSize = 128`
	`2458`	`+ val line = "X" * (bufSize - 1) + "\| \|"`
	`2459`	`+ withTempPath { path =>`
	`2460`	`+ Seq(line).toDF.write.text(path.getAbsolutePath)`
	`2461`	`+ assert(spark.read.format("csv")`
	`2462`	`+ .option("delimiter", "\|")`
	`2463`	`+ .option("ignoreTrailingWhiteSpace", "true").load(path.getAbsolutePath).count() == 1)`
	`2464`	`+ }`
	`2465`	`+ }`
`2455`	`2466`	`}`
`2456`	`2467`
`2457`	`2468`	`class CSVv1Suite extends CSVSuite {`