Skip to content

Commit 801755b

Browse files
committed
test: assert the narrow Iceberg data file omits the nested fields
1 parent baf26a1 commit 801755b

1 file changed

Lines changed: 20 additions & 0 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2274,6 +2274,26 @@ class CometIcebergNativeSuite
22742274
.append()
22752275
}
22762276

2277+
// Check the footers, so the test fails if a writer starts filling in the missing fields.
2278+
val fileLeaves = spark.sql(s"SELECT file_path FROM $table.files").collect().map { row =>
2279+
val reader = org.apache.parquet.hadoop.ParquetFileReader.open(
2280+
org.apache.parquet.hadoop.util.HadoopInputFile.fromPath(
2281+
new org.apache.hadoop.fs.Path(row.getString(0)),
2282+
spark.sessionState.newHadoopConf()))
2283+
try {
2284+
reader.getFooter.getFileMetaData.getSchema.getColumns.asScala
2285+
.map(_.getPath.mkString("."))
2286+
.toSet
2287+
} finally {
2288+
reader.close()
2289+
}
2290+
}
2291+
val omitted = Set("s.b", "l.list.element.y")
2292+
// Check the paths against the full-schema file first, or a wrong path makes the next
2293+
// assert always pass.
2294+
assert(fileLeaves.exists(omitted.subsetOf), s"leaves: ${fileLeaves.mkString(", ")}")
2295+
assert(fileLeaves.exists(!_.exists(omitted)), s"leaves: ${fileLeaves.mkString(", ")}")
2296+
22772297
checkPrunedAndFullReads(s"SELECT * FROM $table ORDER BY id")
22782298
checkPrunedAndFullReads(s"SELECT id, s.b FROM $table WHERE s.b IS NULL ORDER BY id")
22792299

0 commit comments

Comments
 (0)