@@ -22,7 +22,6 @@ package org.apache.comet.exec
2222import java .{util => ju }
2323import java .nio .charset .StandardCharsets
2424
25- import scala .collection .mutable
2625import scala .jdk .CollectionConverters ._
2726
2827import org .apache .arrow .compression .ZstdCompressionCodec
@@ -1998,12 +1997,12 @@ class CometInMemoryCacheSuite extends CometTestBase {
19981997 }
19991998
20001999 test(" Comet in-memory cache records per-column decoded sizes in its statistics" ) {
2001- // SimpleMetricsCachedBatch reserves a fifth field per column for its size and sums those into
2002- // the batch's sizeInBytes, which is what Spark's planner reads as the size of a materialized
2003- // cached relation. Spark's own formats record a column's decoded size there, so each field is
2004- // compared with its column decoded back out of the payload, not with what the column occupies
2005- // compressed. Run over the nested relation as well: a nested column's size is the sum of its
2006- // whole subtree, so this is also where a size attributed to the wrong column surfaces .
2000+ // Each column's size field must be its decoded size (see ArrowCachedBatchSerializer.statsRow),
2001+ // so each field is compared with its column decoded back out of the payload, not with what the
2002+ // column occupies compressed. Run over the nested relation as well: a nested column's size is
2003+ // the sum of its whole subtree, so this is also where a size attributed to the wrong column
2004+ // surfaces. And over the dictionary relation, whose columns reach the writer dictionary
2005+ // encoded: a size measured before they are decoded would be the size of their indices .
20072006 def checkSizes (
20082007 relation : org.apache.spark.sql.execution.columnar.InMemoryRelation ,
20092008 batches : Array [CachedBatch ]): Unit = {
@@ -2028,14 +2027,15 @@ class CometInMemoryCacheSuite extends CometTestBase {
20282027
20292028 withProjectionCache(checkSizes _)
20302029 withNestedProjectionCache(checkSizes _)
2030+ withDictionaryCache(relation =>
2031+ checkSizes(relation, relation.cacheBuilder.cachedColumnBuffers.collect()))
20312032 }
20322033
20332034 test(" Comet in-memory cache reports the same relation size under every codec" ) {
2034- // Broadcast thresholds and the shuffled hash join build side are compared against this size.
2035- // Reported compressed, a relation that zstd shrinks several times over would be broadcast
2036- // where the same relation in Spark's cache format is shuffled, so it must not depend on the
2037- // codec.
2038- val sizes = mutable.ArrayBuffer .empty[(String , Long , Long )]
2035+ // The planner compares this size with broadcast thresholds and with the other side of a
2036+ // shuffled hash join, so it must not depend on the codec (see
2037+ // ArrowCachedBatchSerializer.statsRow).
2038+ val measured = Seq .newBuilder[(String , Long , Long )]
20392039 Seq (" zstd" , " none" ).foreach { codec =>
20402040 withSQLConf(
20412041 SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
@@ -2054,11 +2054,12 @@ class CometInMemoryCacheSuite extends CometTestBase {
20542054 .cachedRepresentation
20552055 val batches = relation.cacheBuilder.cachedColumnBuffers.collect()
20562056 val payload = batches.map(CometCachedBatchHelper .payloadSize).sum
2057- sizes += ((codec, relation.computeStats().sizeInBytes.toLong, payload))
2057+ measured += ((codec, relation.computeStats().sizeInBytes.toLong, payload))
20582058 spark.catalog.clearCache()
20592059 }
20602060 }
20612061
2062+ val sizes = measured.result()
20622063 val (_, zstdSize, zstdPayload) = sizes(0 )
20632064 val (_, plainSize, _) = sizes(1 )
20642065 assert(zstdPayload * 2 < zstdSize, s " zstd should compress this relation: $sizes" )
0 commit comments