Skip to content

Commit 05e52b6

Browse files
fjh100456fjh100456
committed
Change compressionCodecClassName to compressionCodecName
1 parent 1b087df commit 05e52b6

File tree

7 files changed

+14
-15
lines changed

7 files changed

+14
-15
lines changed

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ class OrcFileFormat
9494

9595
conf.set(MAPRED_OUTPUT_SCHEMA.getAttribute, dataSchema.catalogString)
9696

97-
conf.set(COMPRESS.getAttribute, orcOptions.compressionCodecClassName)
97+
conf.set(COMPRESS.getAttribute, orcOptions.compressionCodecName)
9898

9999
conf.asInstanceOf[JobConf]
100100
.setOutputFormat(classOf[org.apache.orc.mapred.OrcOutputFormat[OrcStruct]])

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOptions.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ class OrcOptions(
4141
* Compression codec to use.
4242
* Acceptable values are defined in [[shortOrcCompressionCodecNames]].
4343
*/
44-
val compressionCodecClassName: String = {
44+
val compressionCodecName: String = {
4545
// `compression`, `orc.compress`(i.e., OrcConf.COMPRESS), and `spark.sql.orc.compression.codec`
4646
// are in order of precedence from highest to lowest.
4747
val orcCompressionConf = parameters.get(COMPRESS.getAttribute)

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ class ParquetFileFormat
123123
sparkSession.sessionState.conf.parquetOutputTimestampType.toString)
124124

125125
// Sets compression scheme
126-
conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecClassName)
126+
conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecName)
127127

128128
// SPARK-15719: Disables writing Parquet summary files by default.
129129
if (conf.get(ParquetOutputFormat.ENABLE_JOB_SUMMARY) == null) {

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetOptions.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
2828
/**
2929
* Options for the Parquet data source.
3030
*/
31-
private[parquet] class ParquetOptions(
31+
class ParquetOptions(
3232
@transient private val parameters: CaseInsensitiveMap[String],
3333
@transient private val sqlConf: SQLConf)
3434
extends Serializable {
@@ -42,7 +42,7 @@ private[parquet] class ParquetOptions(
4242
* Compression codec to use. By default use the value specified in SQLConf.
4343
* Acceptable values are defined in [[shortParquetCompressionCodecNames]].
4444
*/
45-
val compressionCodecClassName: String = {
45+
val compressionCodecName: String = {
4646
// `compression`, `parquet.compression`(i.e., ParquetOutputFormat.COMPRESSION), and
4747
// `spark.sql.parquet.compression.codec`
4848
// are in order of precedence from highest to lowest.
@@ -76,7 +76,7 @@ object ParquetOptions {
7676
val MERGE_SCHEMA = "mergeSchema"
7777

7878
// The parquet compression short names
79-
private val shortParquetCompressionCodecNames = Map(
79+
val shortParquetCompressionCodecNames = Map(
8080
"none" -> CompressionCodecName.UNCOMPRESSED,
8181
"uncompressed" -> CompressionCodecName.UNCOMPRESSED,
8282
"snappy" -> CompressionCodecName.SNAPPY,

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -134,30 +134,29 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
134134
test("SPARK-18433: Improve DataSource option keys to be more case-insensitive") {
135135
val conf = spark.sessionState.conf
136136
val option = new OrcOptions(Map(COMPRESS.getAttribute.toUpperCase(Locale.ROOT) -> "NONE"), conf)
137-
assert(option.compressionCodecClassName == "NONE")
137+
assert(option.compressionCodecName == "NONE")
138138
}
139139

140140
test("SPARK-21839: Add SQL config for ORC compression") {
141141
val conf = spark.sessionState.conf
142142
// Test if the default of spark.sql.orc.compression.codec is snappy
143-
assert(new OrcOptions(Map.empty[String, String], conf).compressionCodecClassName == "SNAPPY")
143+
assert(new OrcOptions(Map.empty[String, String], conf).compressionCodecName == "SNAPPY")
144144

145145
// OrcOptions's parameters have a higher priority than SQL configuration.
146146
// `compression` -> `orc.compression` -> `spark.sql.orc.compression.codec`
147147
withSQLConf(SQLConf.ORC_COMPRESSION.key -> "uncompressed") {
148-
assert(new OrcOptions(Map.empty[String, String], conf).compressionCodecClassName == "NONE")
148+
assert(new OrcOptions(Map.empty[String, String], conf).compressionCodecName == "NONE")
149149
val map1 = Map(COMPRESS.getAttribute -> "zlib")
150150
val map2 = Map(COMPRESS.getAttribute -> "zlib", "compression" -> "lzo")
151-
assert(new OrcOptions(map1, conf).compressionCodecClassName == "ZLIB")
152-
assert(new OrcOptions(map2, conf).compressionCodecClassName == "LZO")
151+
assert(new OrcOptions(map1, conf).compressionCodecName == "ZLIB")
152+
assert(new OrcOptions(map2, conf).compressionCodecName == "LZO")
153153
}
154154

155155
// Test all the valid options of spark.sql.orc.compression.codec
156156
Seq("NONE", "UNCOMPRESSED", "SNAPPY", "ZLIB", "LZO").foreach { c =>
157157
withSQLConf(SQLConf.ORC_COMPRESSION.key -> c) {
158158
val expected = if (c == "UNCOMPRESSED") "NONE" else c
159-
assert(
160-
new OrcOptions(Map.empty[String, String], conf).compressionCodecClassName == expected)
159+
assert(new OrcOptions(Map.empty[String, String], conf).compressionCodecName == expected)
161160
}
162161
}
163162
}

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -768,7 +768,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
768768
test("SPARK-18433: Improve DataSource option keys to be more case-insensitive") {
769769
withSQLConf(SQLConf.PARQUET_COMPRESSION.key -> "snappy") {
770770
val option = new ParquetOptions(Map("Compression" -> "uncompressed"), spark.sessionState.conf)
771-
assert(option.compressionCodecClassName == "UNCOMPRESSED")
771+
assert(option.compressionCodecName == "UNCOMPRESSED")
772772
}
773773
}
774774
}

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable
7474

7575
val configuration = job.getConfiguration
7676

77-
configuration.set(COMPRESS.getAttribute, orcOptions.compressionCodecClassName)
77+
configuration.set(COMPRESS.getAttribute, orcOptions.compressionCodecName)
7878
configuration match {
7979
case conf: JobConf =>
8080
conf.setOutputFormat(classOf[OrcOutputFormat])

0 commit comments

Comments
 (0)