2020package org .apache .comet .parquet
2121
2222import java .io .File
23+ import java .nio .charset .Charset
2324import java .util .concurrent .atomic .AtomicReference
2425
2526import scala .util .control .NonFatal
@@ -72,12 +73,15 @@ abstract class CometParquetWriterTestBase extends CometTestBase {
7273 * [[RecordingFileSystem ]], whose recordings start empty and are cleared afterwards.
7374 *
7475 * The output directory's name has a space, `%`, `#`, `?` and a non-ASCII character, which a
75- * path has to keep as it passes from the commit protocol to Hadoop.
76+ * path has to keep as it passes from the commit protocol to Hadoop. The character is left out
77+ * when the JVM cannot encode it in a file name, as in a container with the POSIX locale, where
78+ * even the job's own cleanup of the directory fails.
7679 */
7780 protected def withRecordingFileSystem (source : => DataFrame )(
7881 write : (DataFrame , String ) => Unit ): Unit =
7982 withTempPath { dir =>
80- val outputPath = new File (dir, " output dir %#?\u00e9 .parquet" ).getAbsolutePath
83+ val nonAscii = if (canEncodeInFileName('\u00e9 ' )) " \u00e9 " else " "
84+ val outputPath = new File (dir, s " output dir %#? $nonAscii.parquet " ).getAbsolutePath
8185 val sourcePath = new File (dir, " source.parquet" ).getAbsolutePath
8286 withNativeWriter {
8387 val df = materializeAsCometSource(source, sourcePath)
@@ -97,6 +101,12 @@ abstract class CometParquetWriterTestBase extends CometTestBase {
97101 }
98102 }
99103
104+ /** Whether the JVM can name a file with `c`, which depends on `sun.jnu.encoding`. */
105+ private def canEncodeInFileName (c : Char ): Boolean = {
106+ val encoding = System .getProperty(" sun.jnu.encoding" , " UTF-8" )
107+ Charset .isSupported(encoding) && Charset .forName(encoding).newEncoder().canEncode(c)
108+ }
109+
100110 protected def nativeWriteAllowIncompatKey : String =
101111 if (isSpark40Plus) {
102112 CometConf .COMET_OPERATOR_WRITE_FILES_ALLOW_INCOMPAT .key
0 commit comments