@@ -30,7 +30,7 @@ import org.apache.spark.sql.comet.execution.arrow.{ArrowCachedBatchSerializer, C
3030import org .apache .spark .sql .execution .ColumnarToRowExec
3131import org .apache .spark .sql .execution .adaptive .AdaptiveSparkPlanExec
3232import org .apache .spark .sql .execution .aggregate .HashAggregateExec
33- import org .apache .spark .sql .execution .columnar .{CometInMemoryRelationHelper , DefaultCachedBatchSerializer , InMemoryRelation , InMemoryTableScanExec }
33+ import org .apache .spark .sql .execution .columnar .{CometInMemoryRelationHelper , DefaultCachedBatch , DefaultCachedBatchSerializer , InMemoryRelation , InMemoryTableScanExec }
3434import org .apache .spark .sql .execution .vectorized .OnHeapColumnVector
3535import org .apache .spark .sql .internal .{SQLConf , StaticSQLConf }
3636import org .apache .spark .sql .types .{DataType , LongType , StringType }
@@ -238,6 +238,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase {
238238 scanned = 6 )
239239
240240 runCodecBenchmark(flatRelation)
241+ runBuildBenchmark(flatRelation)
241242 runSparkOperatorBenchmark(flatRelation)
242243 runAdaptiveBenchmark(flatRelation)
243244 }
@@ -347,6 +348,90 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase {
347348 }
348349 }
349350
351+ /**
352+ * Building the cache in each format, and the footprint each format leaves, against Spark's own
353+ * cache format. Turning the feature on changes how every cached relation is written, and a job
354+ * that caches a relation to read it only once or twice pays more for the build than for the
355+ * reads.
356+ *
357+ * What the build costs depends on what produces the rows being cached. Comet's format is
358+ * written straight from the Arrow batches of a plan that runs natively, and from rows
359+ * otherwise, while Spark's format is always built from rows, so above Comet operators it pays a
360+ * columnar-to-row transition first. The relation's source stands in for both kinds of input:
361+ * with spark.comet.exec.range.enabled Comet generates and projects its rows natively, as above
362+ * a native scan, and without it the source runs on Spark. Comet and AQE are on and Comet's
363+ * other settings are at their defaults, as in runAdaptiveBenchmark.
364+ *
365+ * Only one copy is cached at a time, as in runCodecBenchmark.
366+ */
367+ private def runBuildBenchmark (relation : CachedRelation ): Unit = {
368+ val view = s " ${relation.table}_build "
369+ val formats = Seq (
370+ (" Spark's cache format" , sparkSerializer, Seq .empty[(String , String )]),
371+ (" Comet's cache format, zstd" , cometSerializer, codecConf(" zstd" )),
372+ (" Comet's cache format, none" , cometSerializer, codecConf(" none" )))
373+ val inputs = Seq (
374+ (" Comet operators" , true , Seq (CometConf .COMET_EXEC_RANGE_ENABLED .key -> " true" )),
375+ (" Spark operators" , false , Seq .empty[(String , String )]))
376+
377+ spark.catalog.clearCache()
378+ withTempTable(view) {
379+ spark
380+ .sql(s " SELECT ${relation.columns.mkString(" , " )} FROM ${relation.source}" )
381+ .createOrReplaceTempView(view)
382+
383+ inputs.foreach { case (input, nativeInput, inputConf) =>
384+ val benchmark =
385+ new Benchmark (
386+ s " in-memory cache build from $input" ,
387+ relation.rows.toLong,
388+ output = output)
389+ val footprints = new Array [Long ](formats.length)
390+ formats.zipWithIndex.foreach { case ((name, serializer, formatConf), i) =>
391+ // Timed around the caching alone: dropping the previous copy is setup, as in
392+ // runCodecBenchmark.
393+ benchmark.addTimerCase(name) { timer =>
394+ spark.catalog.uncacheTable(view)
395+ withCacheSerializer(serializer) {
396+ withSQLConf(adaptiveConf ++ inputConf ++ formatConf : _* ) {
397+ timer.startTiming()
398+ spark.catalog.cacheTable(view)
399+ spark.table(view).count()
400+ timer.stopTiming()
401+ }
402+ }
403+ // On the case's first call, which is a warmup iteration.
404+ if (footprints(i) == 0L ) {
405+ verifyBuild(view, serializer, nativeInput)
406+ footprints(i) = cachedBytes(view)
407+ }
408+ }
409+ }
410+ benchmark.run()
411+
412+ // As in runCodecBenchmark, footprint has no column in a Benchmark table.
413+ formats.zip(footprints).foreach { case ((name, _, _), bytes) =>
414+ benchmark.out.println(f " Cached footprint ( $name): ${bytes / (1024.0 * 1024.0 )}%.1f MiB " )
415+ }
416+ }
417+
418+ spark.catalog.uncacheTable(view)
419+ }
420+ }
421+
422+ // Pins what a build case claims: the named serializer cached the relation, from a plan that ran
423+ // natively exactly when the case says its input came from Comet operators.
424+ private def verifyBuild (view : String , serializer : String , nativeInput : Boolean ): Unit = {
425+ val relation = cachedRelation(view)
426+ val actual = relation.cacheBuilder.serializer.getClass.getName
427+ assert(actual == serializer, s " Expected a relation cached by $serializer, not $actual" )
428+ val plan = relation.cacheBuilder.cachedPlan
429+ val cometOperators = collect(plan) { case p if p.nodeName.startsWith(" Comet" ) => p }
430+ assert(
431+ cometOperators.nonEmpty == nativeInput,
432+ s " Expected the cached plan to run ${if (nativeInput) " natively" else " on Spark" }: \n $plan" )
433+ }
434+
350435 /**
351436 * Reads that feed Spark operators rather than Comet ones, against Spark's own cache format.
352437 *
@@ -642,20 +727,26 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase {
642727
643728 /** What the cached relation behind `view` occupies, summed over its batches as written. */
644729 private def cachedBytes (view : String ): Long = {
645- val relation = spark
646- .table(view)
647- .queryExecution
648- .optimizedPlan
649- .collectFirst { case r : InMemoryRelation => r }
650- .getOrElse(sys.error(s " $view is not cached " ))
730+ val relation = cachedRelation(view)
651731 // The stored payloads rather than computeStats, which reports the relation's decoded size and
652732 // so is the same for every codec.
653733 assert(relation.cacheBuilder.isCachedColumnBuffersLoaded, s " $view is not materialized " )
654734 relation.cacheBuilder.cachedColumnBuffers
655- .map(CometCachedBatchHelper .payloadSize)
735+ .map {
736+ case batch : DefaultCachedBatch => batch.buffers.map(_.length.toLong).sum
737+ case batch => CometCachedBatchHelper .payloadSize(batch)
738+ }
656739 .fold(0L )(_ + _)
657740 }
658741
742+ private def cachedRelation (view : String ): InMemoryRelation =
743+ spark
744+ .table(view)
745+ .queryExecution
746+ .optimizedPlan
747+ .collectFirst { case r : InMemoryRelation => r }
748+ .getOrElse(sys.error(s " $view is not cached " ))
749+
659750 private def runStatsBenchmark (): Unit = {
660751 val batchSize = 10000
661752 val types : Seq [DataType ] = Seq .fill(3 )(LongType ) ++ Seq .fill(3 )(StringType )
0 commit comments