1616// under the License.
1717
1818use crate :: cache:: default_cache:: DefaultCache ;
19- pub use crate :: cache:: {
20- Cache , CacheValue , FileStatisticsCacheKey , SchemaFingerprint , TableScopedPath ,
21- } ;
19+ pub use crate :: cache:: { Cache , CacheValue , SchemaFingerprint , TableScopedPath } ;
2220use datafusion_common:: HashMap ;
2321use datafusion_common:: heap_size:: { DFHeapSize , DFHeapSizeCtx } ;
2422use datafusion_common:: { Result , Statistics } ;
@@ -52,11 +50,12 @@ pub const DEFAULT_METADATA_CACHE_LIMIT: usize = 50 * 1024 * 1024; // 50M
5250///
5351/// The typical usage pattern is:
5452/// 1. Call `get(path)` to check for cached value
55- /// 2. If `Some(cached)`, validate with `cached.is_valid_for(¤t_meta)`
53+ /// 2. If `Some(cached)`, validate with
54+ /// `cached.is_valid_for(¤t_meta, ¤t_schema_fingerprint)`
5655/// 3. If invalid or missing, compute new value and call `put(path, new_value)`
5756///
5857/// See [`crate::runtime_env::RuntimeEnv`] for more details
59- pub type FileStatisticsCache = dyn Cache < FileStatisticsCacheKey , CachedFileMetadata > ;
58+ pub type FileStatisticsCache = dyn Cache < TableScopedPath , CachedFileMetadata > ;
6059
6160/// A cache for storing the [`ObjectMeta`]s that result from listing a path.
6261///
@@ -93,11 +92,13 @@ pub type FileMetadataCache = dyn Cache<Path, CachedFileMetadataEntry>;
9392/// Cached metadata for a file, including statistics and ordering.
9493///
9594/// This struct embeds the [`ObjectMeta`] used for cache validation,
96- /// along with the cached statistics and ordering information.
95+ /// the `file_schema` fingerprint, cached statistics, and ordering information.
9796#[ derive( Debug , Clone , PartialEq , Eq ) ]
9897pub struct CachedFileMetadata {
9998 /// File metadata used for cache validation (size, last_modified).
10099 pub meta : ObjectMeta ,
100+ /// Fingerprint of the `file_schema` used to compute `statistics`.
101+ pub schema_fingerprint : Arc < SchemaFingerprint > ,
101102 /// Cached statistics for the file, if available.
102103 pub statistics : Arc < Statistics > ,
103104 /// Cached ordering for the file.
@@ -108,22 +109,29 @@ impl CachedFileMetadata {
108109 /// Create a new cached file metadata entry.
109110 pub fn new (
110111 meta : ObjectMeta ,
112+ schema_fingerprint : Arc < SchemaFingerprint > ,
111113 statistics : Arc < Statistics > ,
112114 ordering : Option < LexOrdering > ,
113115 ) -> Self {
114116 Self {
115117 meta,
118+ schema_fingerprint,
116119 statistics,
117120 ordering,
118121 }
119122 }
120123
121124 /// Check if this cached entry is still valid for the given metadata.
122125 ///
123- /// Returns true if the file size and last modified time match.
124- pub fn is_valid_for ( & self , current_meta : & ObjectMeta ) -> bool {
126+ /// Returns true if the file size, last modified time, and schema match.
127+ pub fn is_valid_for (
128+ & self ,
129+ current_meta : & ObjectMeta ,
130+ current_schema_fingerprint : & SchemaFingerprint ,
131+ ) -> bool {
125132 self . meta . size == current_meta. size
126133 && self . meta . last_modified == current_meta. last_modified
134+ && self . schema_fingerprint . as_ref ( ) == current_schema_fingerprint
127135 }
128136}
129137
@@ -141,6 +149,8 @@ impl DFHeapSize for CachedFileMetadata {
141149 + self . meta . e_tag . heap_size ( ctx)
142150 + self . meta . location . as_ref ( ) . heap_size ( ctx)
143151 + self . statistics . heap_size ( ctx)
152+ // Do not deep-count `schema_fingerprint`: each ListingTable shares one
153+ // fingerprint across all cached files.
144154 //TODO add ordering once LexOrdering/PhysicalExpr implements DFHeapSize
145155 }
146156}
@@ -307,7 +317,7 @@ impl CacheManager {
307317 Some ( Arc :: clone ( fsc) )
308318 }
309319 None if config. file_statistics_cache_limit > 0 => Some ( Arc :: new (
310- DefaultCache :: < FileStatisticsCacheKey , CachedFileMetadata > :: new (
320+ DefaultCache :: < TableScopedPath , CachedFileMetadata > :: new (
311321 config. file_statistics_cache_limit ,
312322 )
313323 . with_name ( "DefaultFileStatisticsCache" ) ,
0 commit comments