Skip to content

Commit 223e53c

Browse files
alexey-milovidovalexey-milovidov
committed
Revert "Merge pull request #89750 from grantholly-clickhouse/parquet_footer_cache"
This reverts commit 5243ef7, reversing changes made to 6e3a5ad.
1 parent 41cc63c commit 223e53c

35 files changed

+49
-665
lines changed

‎docs/en/sql-reference/statements/system.md‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,6 @@ Clears the mark cache.
101101

102102
Clears the iceberg metadata cache.
103103

104-
## SYSTEM DROP PARQUET METADATA CACHE {#drop-parquet-metadata-cache}
105-
106-
Clears the parquet metadata cache.
107-
108104
## SYSTEM CLEAR|DROP TEXT INDEX CACHES {#drop-text-index-caches}
109105

110106
Clears the text index's header, dictionary and postings caches.

‎programs/local/LocalServer.cpp‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,6 @@ namespace ServerSetting
123123
extern const ServerSettingsUInt64 iceberg_metadata_files_cache_size;
124124
extern const ServerSettingsUInt64 iceberg_metadata_files_cache_max_entries;
125125
extern const ServerSettingsDouble iceberg_metadata_files_cache_size_ratio;
126-
extern const ServerSettingsString parquet_metadata_cache_policy;
127-
extern const ServerSettingsUInt64 parquet_metadata_cache_size;
128-
extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries;
129-
extern const ServerSettingsDouble parquet_metadata_cache_size_ratio;
130126
extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size;
131127
extern const ServerSettingsUInt64 max_io_thread_pool_free_size;
132128
extern const ServerSettingsUInt64 max_io_thread_pool_size;
@@ -989,18 +985,6 @@ void LocalServer::processConfig()
989985
}
990986
global_context->setIcebergMetadataFilesCache(iceberg_metadata_files_cache_policy, iceberg_metadata_files_cache_size, iceberg_metadata_files_cache_max_entries, iceberg_metadata_files_cache_size_ratio);
991987
#endif
992-
#if USE_PARQUET
993-
String parquet_metadata_cache_policy = server_settings[ServerSetting::parquet_metadata_cache_policy];
994-
size_t parquet_metadata_cache_size = server_settings[ServerSetting::parquet_metadata_cache_size];
995-
size_t parquet_metadata_cache_max_entries = server_settings[ServerSetting::parquet_metadata_cache_max_entries];
996-
double parquet_metadata_cache_size_ratio = server_settings[ServerSetting::parquet_metadata_cache_size_ratio];
997-
if (parquet_metadata_cache_size > max_cache_size)
998-
{
999-
parquet_metadata_cache_size = max_cache_size;
1000-
LOG_INFO(log, "Lowered Parquet metadata cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(parquet_metadata_cache_size));
1001-
}
1002-
global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio);
1003-
#endif
1004988

1005989
Names allowed_disks_table_engines;
1006990
splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value);

‎programs/server/Server.cpp‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -262,10 +262,6 @@ namespace ServerSetting
262262
extern const ServerSettingsUInt64 iceberg_metadata_files_cache_size;
263263
extern const ServerSettingsUInt64 iceberg_metadata_files_cache_max_entries;
264264
extern const ServerSettingsDouble iceberg_metadata_files_cache_size_ratio;
265-
extern const ServerSettingsString parquet_metadata_cache_policy;
266-
extern const ServerSettingsUInt64 parquet_metadata_cache_size;
267-
extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries;
268-
extern const ServerSettingsDouble parquet_metadata_cache_size_ratio;
269265
extern const ServerSettingsUInt64 io_thread_pool_queue_size;
270266
extern const ServerSettingsBool jemalloc_enable_global_profiler;
271267
extern const ServerSettingsBool jemalloc_collect_global_profile_samples_in_trace_log;
@@ -2038,18 +2034,6 @@ try
20382034
}
20392035
global_context->setIcebergMetadataFilesCache(iceberg_metadata_files_cache_policy, iceberg_metadata_files_cache_size, iceberg_metadata_files_cache_max_entries, iceberg_metadata_files_cache_size_ratio);
20402036
#endif
2041-
#if USE_PARQUET
2042-
String parquet_metadata_cache_policy = server_settings[ServerSetting::parquet_metadata_cache_policy];
2043-
size_t parquet_metadata_cache_size = server_settings[ServerSetting::parquet_metadata_cache_size];
2044-
size_t parquet_metadata_cache_max_entries = server_settings[ServerSetting::parquet_metadata_cache_max_entries];
2045-
double parquet_metadata_cache_size_ratio = server_settings[ServerSetting::parquet_metadata_cache_size_ratio];
2046-
if (parquet_metadata_cache_size > max_cache_size)
2047-
{
2048-
parquet_metadata_cache_size = max_cache_size;
2049-
LOG_INFO(log, "Lowered Parquet metadata cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(parquet_metadata_cache_size));
2050-
}
2051-
global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio);
2052-
#endif
20532037

20542038
Names allowed_disks_table_engines;
20552039
splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value);

‎src/Access/Common/AccessType.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,6 @@ enum class AccessType : uint8_t
315315
M(SYSTEM_PREWARM_MARK_CACHE, "SYSTEM PREWARM MARK, PREWARM MARK CACHE, PREWARM MARKS", GLOBAL, SYSTEM_DROP_CACHE) \
316316
M(SYSTEM_DROP_MARK_CACHE, "SYSTEM CLEAR MARK CACHE, SYSTEM DROP MARK, DROP MARK CACHE, DROP MARKS", GLOBAL, SYSTEM_DROP_CACHE) \
317317
M(SYSTEM_DROP_ICEBERG_METADATA_CACHE, "SYSTEM CLEAR ICEBERG_METADATA_CACHE, SYSTEM DROP ICEBERG_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
318-
M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
319318
M(SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, "SYSTEM PREWARM PRIMARY INDEX, PREWARM PRIMARY INDEX CACHE, PREWARM PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \
320319
M(SYSTEM_DROP_PRIMARY_INDEX_CACHE, "SYSTEM CLEAR PRIMARY INDEX CACHE, SYSTEM DROP PRIMARY INDEX, DROP PRIMARY INDEX CACHE, DROP PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \
321320
M(SYSTEM_DROP_UNCOMPRESSED_CACHE, "SYSTEM CLEAR UNCOMPRESSED CACHE, SYSTEM DROP UNCOMPRESSED, DROP UNCOMPRESSED CACHE, DROP UNCOMPRESSED", GLOBAL, SYSTEM_DROP_CACHE) \

‎src/Common/CurrentMetrics.cpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -318,8 +318,6 @@
318318
M(AsyncInsertCacheSize, "Number of async insert hash id in cache") \
319319
M(IcebergMetadataFilesCacheBytes, "Size of the Iceberg metadata cache in bytes") \
320320
M(IcebergMetadataFilesCacheFiles, "Number of cached files in the Iceberg metadata cache") \
321-
M(ParquetMetadataCacheBytes, "Size of the Parquet metadata cache in bytes") \
322-
M(ParquetMetadataCacheFiles, "Number of cached files in the Parquet metadata cache") \
323321
M(AvroSchemaCacheBytes, "Size of the Avro schema cache in bytes") \
324322
M(AvroSchemaCacheCells, "Number of cached Avro schemas") \
325323
M(AvroSchemaRegistryCacheBytes, "Size of the Avro schema registry cache in bytes") \

‎src/Common/ProfileEvents.cpp‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,9 +98,6 @@
9898
M(IcebergMetadataFilesCacheMisses, "Number of times iceberg metadata files have not been found in the iceberg metadata cache and had to be read from (remote) disk.", ValueType::Number) \
9999
M(IcebergMetadataFilesCacheWeightLost, "Approximate number of bytes evicted from the iceberg metadata cache.", ValueType::Number) \
100100
M(IcebergMetadataReadWaitTimeMicroseconds, "Total time data readers spend waiting for iceberg metadata files to be read and parsed, summed across all reader threads.", ValueType::Microseconds) \
101-
M(ParquetMetadataCacheHits, "Number of times parquet metadata has been found in the cache.", ValueType::Number) \
102-
M(ParquetMetadataCacheMisses, "Number of times parquet metadata has not been found in the cache and had to be read from disk.", ValueType::Number) \
103-
M(ParquetMetadataCacheWeightLost, "Approximate number of bytes evicted from the parquet metadata cache.", ValueType::Number) \
104101
M(IcebergIteratorInitializationMicroseconds, "Total time spent on synchronous initialization of iceberg data iterators.", ValueType::Microseconds) \
105102
M(IcebergMetadataUpdateMicroseconds, "Total time spent on synchronous initialization of iceberg data iterators.", ValueType::Microseconds) \
106103
M(IcebergMetadataReturnedObjectInfos, "Total number of returned object infos from iceberg iterator.", ValueType::Number) \

‎src/Core/Defines.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,6 @@ static constexpr auto DEFAULT_ICEBERG_METADATA_CACHE_POLICY = "SLRU";
119119
static constexpr auto DEFAULT_ICEBERG_METADATA_CACHE_MAX_SIZE = 1_GiB;
120120
static constexpr auto DEFAULT_ICEBERG_METADATA_CACHE_SIZE_RATIO = 0.5;
121121
static constexpr auto DEFAULT_ICEBERG_METADATA_CACHE_MAX_ENTRIES = 1000;
122-
static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_POLICY = "SLRU";
123-
static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE = 512_MiB;
124-
static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO = 0.5;
125-
static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES = 5000;
126122
static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_POLICY = "SLRU";
127123
static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_MAX_SIZE = 100_MiB;
128124
static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_SIZE_RATIO = 0.5l;

‎src/Core/ServerSettings.cpp‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -505,10 +505,6 @@ namespace
505505
DECLARE(UInt64, iceberg_metadata_files_cache_size, DEFAULT_ICEBERG_METADATA_CACHE_MAX_SIZE, "Maximum size of iceberg metadata cache in bytes. Zero means disabled.", 0) \
506506
DECLARE(UInt64, iceberg_metadata_files_cache_max_entries, DEFAULT_ICEBERG_METADATA_CACHE_MAX_ENTRIES, "Maximum size of iceberg metadata files cache in entries. Zero means disabled.", 0) \
507507
DECLARE(Double, iceberg_metadata_files_cache_size_ratio, DEFAULT_ICEBERG_METADATA_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the iceberg metadata cache relative to the cache's total size.", 0) \
508-
DECLARE(String, parquet_metadata_cache_policy, DEFAULT_PARQUET_METADATA_CACHE_POLICY, "Parquet metadata cache policy name.", 0) \
509-
DECLARE(UInt64, parquet_metadata_cache_size, DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE, "Maximum size of parquet metadata cache in bytes. Zero means disabled.", 0) \
510-
DECLARE(UInt64, parquet_metadata_cache_max_entries, DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES, "Maximum size of parquet metadata files cache in entries. Zero means disabled.", 0) \
511-
DECLARE(Double, parquet_metadata_cache_size_ratio, DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the parquet metadata cache relative to the cache's total size.", 0) \
512508
DECLARE(String, allowed_disks_for_table_engines, "", "List of disks allowed for use with Iceberg", 0) \
513509
DECLARE(String, vector_similarity_index_cache_policy, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_POLICY, "Vector similarity index cache policy name.", 0) \
514510
DECLARE(UInt64, vector_similarity_index_cache_size, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_MAX_SIZE, R"(Size of cache for vector similarity indexes. Zero means disabled.

‎src/Core/Settings.cpp‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5159,14 +5159,6 @@ If turned on, iceberg table function and iceberg storage may utilize the iceberg
51595159
51605160
Possible values:
51615161
5162-
- 0 - Disabled
5163-
- 1 - Enabled
5164-
)", 0) \
5165-
DECLARE(Bool, use_parquet_metadata_cache, true, R"(
5166-
If turned on, parquet format may utilize the parquet metadata cache.
5167-
5168-
Possible values:
5169-
51705162
- 0 - Disabled
51715163
- 1 - Enabled
51725164
)", 0) \

‎src/Core/SettingsChangesHistory.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
4444
{"deduplicate_blocks_in_dependent_materialized_views", false, true, "Enable deduplication for dependent materialized views by default."},
4545
{"deduplicate_insert", "backward_compatible_choice", "enable", "Enable deduplication for all sync and async inserts by default."},
4646
{"use_parquet_metadata_cache", true, true, "Enables cache of parquet file metadata."},
47+
{"deduplicate_insert", "backward_compatible_choice", "backward_compatible_choice", "New setting to control deduplication for INSERT queries."},
4748
{"enable_join_runtime_filters", false, true, "Enabled this optimization"},
4849
{"parallel_replicas_filter_pushdown", false, false, "New setting"},
4950
{"enable_automatic_decision_for_merging_across_partitions_for_final", true, true, "New setting"},

0 commit comments

Comments
 (0)