Skip to content

Commit ec13f59

Browse files
committed
perf(cache): fast-path schema fingerprint validation
Use Arc pointer identity to avoid deep schema comparison on common file statistics cache hits. Keep exact equality as a fallback for equivalent fingerprints built from different Arc allocations. Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 985b8f3 commit ec13f59

3 files changed

Lines changed: 22 additions & 14 deletions

File tree

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -996,7 +996,7 @@ impl ListingTable {
996996
// stats computed under a different file schema.
997997
if let Some(cache) = &self.collected_statistics
998998
&& let Some(cached) = cache.get(&path)
999-
&& cached.is_valid_for(meta, self.file_schema_fingerprint.as_ref())
999+
&& cached.is_valid_for(meta, &self.file_schema_fingerprint)
10001000
{
10011001
// Return cached statistics and ordering
10021002
return Ok((Arc::clone(&cached.statistics), cached.ordering.clone()));

‎datafusion/execution/src/cache/cache_manager.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,13 @@ impl CachedFileMetadata {
127127
pub fn is_valid_for(
128128
&self,
129129
current_meta: &ObjectMeta,
130-
current_schema_fingerprint: &SchemaFingerprint,
130+
current_schema_fingerprint: &Arc<SchemaFingerprint>,
131131
) -> bool {
132132
self.meta.size == current_meta.size
133133
&& self.meta.last_modified == current_meta.last_modified
134-
&& self.schema_fingerprint.as_ref() == current_schema_fingerprint
134+
&& (Arc::ptr_eq(&self.schema_fingerprint, current_schema_fingerprint)
135+
|| self.schema_fingerprint.as_ref()
136+
== current_schema_fingerprint.as_ref())
135137
}
136138
}
137139

‎datafusion/execution/src/cache/default_cache.rs‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -850,22 +850,30 @@ mod tests {
850850
assert!(result.is_some());
851851

852852
let cached = result.unwrap();
853-
assert!(cached.is_valid_for(&meta, schema_fingerprint.as_ref()));
853+
assert!(cached.is_valid_for(&meta, &schema_fingerprint));
854+
855+
let equivalent_schema_fingerprint =
856+
Arc::new(SchemaFingerprint::from_schema(&schema));
857+
assert!(!Arc::ptr_eq(
858+
&schema_fingerprint,
859+
&equivalent_schema_fingerprint
860+
));
861+
assert!(cached.is_valid_for(&meta, &equivalent_schema_fingerprint));
854862

855863
let different_schema = Schema::new(vec![Field::new(
856864
"different_column",
857865
DataType::Timestamp(TimeUnit::Second, None),
858866
false,
859867
)]);
860868
let different_schema_fingerprint =
861-
SchemaFingerprint::from_schema(&different_schema);
869+
Arc::new(SchemaFingerprint::from_schema(&different_schema));
862870
assert!(!cached.is_valid_for(&meta, &different_schema_fingerprint));
863871

864872
// File size changed - validation should fail
865873
let meta2 = create_test_meta("test", 2048);
866874

867875
let cached = cache.get(&path).unwrap();
868-
assert!(!cached.is_valid_for(&meta2, schema_fingerprint.as_ref()));
876+
assert!(!cached.is_valid_for(&meta2, &schema_fingerprint));
869877

870878
// Update with new value
871879
let cached_value2 = CachedFileMetadata::new(
@@ -967,9 +975,7 @@ mod tests {
967975

968976
// Update to add ordering
969977
let mut cached = cache.get(&path).unwrap();
970-
if cached.is_valid_for(&meta, schema_fingerprint.as_ref())
971-
&& cached.ordering.is_none()
972-
{
978+
if cached.is_valid_for(&meta, &schema_fingerprint) && cached.ordering.is_none() {
973979
cached.ordering = Some(ordering());
974980
}
975981
cache.put(&path, cached);
@@ -1009,7 +1015,7 @@ mod tests {
10091015

10101016
let cached = cache.get(&path).unwrap();
10111017
// Should not be valid for new meta
1012-
assert!(!cached.is_valid_for(&meta_v2, schema_fingerprint.as_ref()));
1018+
assert!(!cached.is_valid_for(&meta_v2, &schema_fingerprint));
10131019

10141020
// Compute new value and update
10151021
let new_cached = CachedFileMetadata::new(
@@ -1056,7 +1062,7 @@ mod tests {
10561062

10571063
// Verify cached ordering is valid
10581064
let cached = cache.get(&path).unwrap();
1059-
assert!(cached.is_valid_for(&meta_v1, schema_fingerprint.as_ref()));
1065+
assert!(cached.is_valid_for(&meta_v1, &schema_fingerprint));
10601066
assert!(cached.ordering.is_some());
10611067

10621068
// File modified (size changed)
@@ -1072,7 +1078,7 @@ mod tests {
10721078

10731079
// Cache entry exists but should be invalid for new metadata
10741080
let cached = cache.get(&path).unwrap();
1075-
assert!(!cached.is_valid_for(&meta_v2, schema_fingerprint.as_ref()));
1081+
assert!(!cached.is_valid_for(&meta_v2, &schema_fingerprint));
10761082

10771083
// Cache new version with different ordering
10781084
let ordering_v2 = ordering(); // New ordering instance
@@ -1086,10 +1092,10 @@ mod tests {
10861092

10871093
// Old metadata should be invalid
10881094
let cached = cache.get(&path).unwrap();
1089-
assert!(!cached.is_valid_for(&meta_v1, schema_fingerprint.as_ref()));
1095+
assert!(!cached.is_valid_for(&meta_v1, &schema_fingerprint));
10901096

10911097
// New metadata should be valid
1092-
assert!(cached.is_valid_for(&meta_v2, schema_fingerprint.as_ref()));
1098+
assert!(cached.is_valid_for(&meta_v2, &schema_fingerprint));
10931099
assert!(cached.ordering.is_some());
10941100
}
10951101

0 commit comments

Comments
 (0)