Skip to content

Commit 301def5

Browse files
committed
test: leave the non-ASCII character out of the output path when the JVM cannot encode it
1 parent 4535d1e commit 301def5

1 file changed

Lines changed: 12 additions & 2 deletions

File tree

‎spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterTestBase.scala‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.comet.parquet
2121

2222
import java.io.File
23+
import java.nio.charset.Charset
2324
import java.util.concurrent.atomic.AtomicReference
2425

2526
import scala.util.control.NonFatal
@@ -72,12 +73,15 @@ abstract class CometParquetWriterTestBase extends CometTestBase {
7273
* [[RecordingFileSystem]], whose recordings start empty and are cleared afterwards.
7374
*
7475
* The output directory's name has a space, `%`, `#`, `?` and a non-ASCII character, which a
75-
* path has to keep as it passes from the commit protocol to Hadoop.
76+
* path has to keep as it passes from the commit protocol to Hadoop. The character is left out
77+
* when the JVM cannot encode it in a file name, as in a container with the POSIX locale, where
78+
* even the job's own cleanup of the directory fails.
7679
*/
7780
protected def withRecordingFileSystem(source: => DataFrame)(
7881
write: (DataFrame, String) => Unit): Unit =
7982
withTempPath { dir =>
80-
val outputPath = new File(dir, "output dir %#?\u00e9.parquet").getAbsolutePath
83+
val nonAscii = if (canEncodeInFileName('\u00e9')) "\u00e9" else ""
84+
val outputPath = new File(dir, s"output dir %#?$nonAscii.parquet").getAbsolutePath
8185
val sourcePath = new File(dir, "source.parquet").getAbsolutePath
8286
withNativeWriter {
8387
val df = materializeAsCometSource(source, sourcePath)
@@ -97,6 +101,12 @@ abstract class CometParquetWriterTestBase extends CometTestBase {
97101
}
98102
}
99103

104+
/** Whether the JVM can name a file with `c`, which depends on `sun.jnu.encoding`. */
105+
private def canEncodeInFileName(c: Char): Boolean = {
106+
val encoding = System.getProperty("sun.jnu.encoding", "UTF-8")
107+
Charset.isSupported(encoding) && Charset.forName(encoding).newEncoder().canEncode(c)
108+
}
109+
100110
protected def nativeWriteAllowIncompatKey: String =
101111
if (isSpark40Plus) {
102112
CometConf.COMET_OPERATOR_WRITE_FILES_ALLOW_INCOMPAT.key

0 commit comments

Comments
 (0)