ClickHouse · rschu1ze · Sep 17, 2025 · Aug 24, 2025 · Aug 25, 2025 · Aug 29, 2025
diff --git a/src/Common/HashTable/StringHashTable.h b/src/Common/HashTable/StringHashTable.h
@@ -275,12 +275,19 @@ class StringHashTable : private boost::noncopyable
     {
     }
 
-    StringHashTable(StringHashTable && rhs) noexcept
-        : m1(std::move(rhs.m1))
-        , m2(std::move(rhs.m2))
-        , m3(std::move(rhs.m3))
-        , ms(std::move(rhs.ms))
+    StringHashTable(StringHashTable && rhs) noexcept { *this = std::move(rhs); }
+
+    StringHashTable & operator=(StringHashTable && rhs) noexcept
     {
+        if (this == &rhs)
+            return *this;
+
+        m0 = std::move(rhs.m0);
+        m1 = std::move(rhs.m1);
+        m2 = std::move(rhs.m2);
+        m3 = std::move(rhs.m3);
+        ms = std::move(rhs.ms);
+        return *this;
     }
 
     ~StringHashTable() = default;
@@ -486,6 +493,15 @@ class StringHashTable : private boost::noncopyable
             + ms.getBufferSizeInBytes();
     }
 
+    void clear()
+    {
+        m1.clearHasZero();
+        m1.clear();
+        m2.clear();
+        m3.clear();
+        ms.clear();
+    }
+
     void clearAndShrink()
     {
         m1.clearHasZero();

diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
@@ -93,6 +93,15 @@
     M(VectorSimilarityIndexCacheHits, "Number of times an index granule has been found in the vector index cache.", ValueType::Number) \
     M(VectorSimilarityIndexCacheMisses, "Number of times an index granule has not been found in the vector index cache and had to be read from disk.", ValueType::Number) \
     M(VectorSimilarityIndexCacheWeightLost, "Approximate number of bytes evicted from the vector index cache.", ValueType::Number) \
+    M(TextIndexReadDictionaryBlocks, "Number of times a dictionary block has been read from the text index.", ValueType::Number) \
+    M(TextIndexReadSparseIndexBlocks, "Number of times a sparse index block has been read from the text index.", ValueType::Number) \
+    M(TextIndexReaderTotalMicroseconds, "Total time spent reading the text index.", ValueType::Microseconds) \
+    M(TextIndexReadGranulesMicroseconds, "Total time spent reading and analyzing granules of the text index.", ValueType::Microseconds) \
+    M(TextIndexBloomFilterTrueNegatives, "Number of times a token has been filtered by bloom filter.", ValueType::Number) \
+    M(TextIndexBloomFilterTruePositives, "Number of times a token has passed the bloom filter and has been found in the dictionary.", ValueType::Number) \
+    M(TextIndexBloomFilterFalsePositives, "Number of times a token has passed the bloom filter and has not been found in the dictionary.", ValueType::Number) \
+    M(TextIndexReadPostings, "Number of times a posting list has been read from the text index.", ValueType::Number) \
+    M(TextIndexUsedEmbeddedPostings, "Number of times a posting list embedded in the dictionary has been used.", ValueType::Number) \
     M(QueryConditionCacheHits, "Number of times an entry has been found in the query condition cache (and reading of marks can be skipped). Only updated for SELECT queries with SETTING use_query_condition_cache = 1.", ValueType::Number) \
     M(QueryConditionCacheMisses, "Number of times an entry has not been found in the query condition cache (and reading of mark cannot be skipped). Only updated for SELECT queries with SETTING use_query_condition_cache = 1.", ValueType::Number) \
     M(QueryCacheHits, "Number of times a query result has been found in the query cache (and query computation was avoided). Only updated for SELECT queries with SETTING use_query_cache = 1.", ValueType::Number) \

diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp
@@ -1514,6 +1514,9 @@ Possible values:
 )", 0) \
     DECLARE(Bool, materialize_skip_indexes_on_insert, true, R"(
 If INSERTs build and store skip indexes. If disabled, skip indexes will be build and stored during merges or by explicit MATERIALIZE INDEX
+)", 0) \
+    DECLARE(Bool, text_index_use_bloom_filter, true, R"(
+For testing purposes, enables or disables usage of bloom filter in text index.
 )", 0) \
     DECLARE(Bool, per_part_index_stats, false, R"(
         Logs index statistics per part
@@ -6971,6 +6974,9 @@ Allows defining columns with [statistics](../../engines/table-engines/mergetree-
     DECLARE(Bool, allow_experimental_full_text_index, false, R"(
 If set to true, allow using the experimental text index.
 )", EXPERIMENTAL) \
+    DECLARE(Bool, query_plan_direct_read_from_text_index, true, R"(
+Allow to perform full text search filtering using only the inverted index in query plan.
+)", 0) \
     DECLARE(Bool, allow_experimental_live_view, false, R"(
 Allows creation of a deprecated LIVE VIEW.
 

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
@@ -54,6 +54,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"query_plan_display_internal_aliases", false, false, "New setting"},
             {"query_plan_max_step_description_length", 1000000000, 500, "New setting"},
             {"allow_experimental_delta_lake_writes", false, false, "New setting."},
+            {"text_index_use_bloom_filter", true, true, "New setting."},
+            {"query_plan_direct_read_from_text_index", true, true, "New setting."},
             {"enable_producing_buckets_out_of_order_in_aggregation", false, true, "New setting"},
             {"jemalloc_enable_profiler", false, false, "New setting"},
             {"jemalloc_collect_profile_samples_in_trace_log", false, false, "New setting"},

diff --git a/src/Functions/searchAnyAll.cpp b/src/Functions/searchAnyAll.cpp
@@ -39,36 +39,19 @@ FunctionSearchImpl<SearchTraits>::FunctionSearchImpl(ContextPtr context)
 }
 
 template <class SearchTraits>
-void FunctionSearchImpl<SearchTraits>::trySetGinFilterParameters(const GinFilter::Parameters & params)
+void FunctionSearchImpl<SearchTraits>::setTokenExtractor(std::unique_ptr<ITokenExtractor> new_token_extractor_)
 {
     /// Index parameters can be set multiple times.
     /// This happens exactly in a case that same searchAny/searchAll query is used again.
     /// This is fine because the parameters would be same.
     if (token_extractor != nullptr)
         return;
 
-    if (params.tokenizer == DefaultTokenExtractor::getExternalName())
-        token_extractor = std::make_unique<DefaultTokenExtractor>();
-    else if (params.tokenizer == NgramTokenExtractor::getExternalName())
-    {
-        auto ngrams = params.ngram_size.value_or(DEFAULT_NGRAM_SIZE);
-        if (ngrams < 2 || ngrams > 8)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Ngrams argument of function '{}' should be between 2 and 8, got: {}", name, ngrams);
-        token_extractor = std::make_unique<NgramTokenExtractor>(ngrams);
-    }
-    else if (params.tokenizer == SplitTokenExtractor::getExternalName())
-    {
-        const auto & separators = params.separators.value_or(std::vector<String>{" "});
-        token_extractor = std::make_unique<SplitTokenExtractor>(separators);
-    }
-    else if (params.tokenizer == NoOpTokenExtractor::getExternalName())
-        token_extractor = std::make_unique<NoOpTokenExtractor>();
-    else
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function '{}' supports only tokenizers 'default', 'ngram', 'split', and 'no_op'", name);
+    token_extractor = std::move(new_token_extractor_);
 }
 
 template <class SearchTraits>
-void FunctionSearchImpl<SearchTraits>::trySetSearchTokens(const std::vector<String> & tokens)
+void FunctionSearchImpl<SearchTraits>::setSearchTokens(const std::vector<String> & tokens)
 {
     static constexpr size_t supported_number_of_needles = 64;
 

diff --git a/src/Functions/searchAnyAll.h b/src/Functions/searchAnyAll.h
@@ -40,8 +40,8 @@ class FunctionSearchImpl : public IFunction
     bool useDefaultImplementationForConstants() const override { return true; }
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
 
-    void trySetGinFilterParameters(const GinFilter::Parameters & params);
-    void trySetSearchTokens(const std::vector<String> & tokens);
+    void setTokenExtractor(std::unique_ptr<ITokenExtractor> new_token_extractor_);
+    void setSearchTokens(const std::vector<String> & tokens);
 
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override;
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override;

diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
@@ -27,6 +27,11 @@ class FunctionNode;
 class IDataType;
 using DataTypePtr = std::shared_ptr<const IDataType>;
 
+namespace QueryPlanOptimizations
+{
+    class FullTextMatchingFunctionDAGReplacer;
+}
+
 namespace JSONBuilder
 {
     class JSONMap;
@@ -469,6 +474,8 @@ class ActionsDAG
     UInt64 getHash() const;
     void updateHash(SipHash & hash_state) const;
 
+    friend class QueryPlanOptimizations::FullTextMatchingFunctionDAGReplacer;
+
     /* Create actions which calculate conjunction of selected nodes.
      * Conjunction nodes are assumed to be predicates that will be combined with AND if multiple.
      *

diff --git a/src/Interpreters/BloomFilter.cpp b/src/Interpreters/BloomFilter.cpp
@@ -56,7 +56,7 @@ void BloomFilter::resize(size_t size_)
     filter.resize(words);
 }
 
-bool BloomFilter::find(const char * data, size_t len)
+bool BloomFilter::find(const char * data, size_t len) const
 {
     size_t hash1 = CityHash_v1_0_2::CityHash64WithSeed(data, len, seed);
     size_t hash2 = CityHash_v1_0_2::CityHash64WithSeed(data, len, SEED_GEN_A * seed + SEED_GEN_B);
@@ -132,7 +132,7 @@ void BloomFilter::addHashWithSeed(const UInt64 & hash, const UInt64 & hash_seed)
     filter[pos / word_bits] |= (1ULL << (pos % word_bits));
 }
 
-bool BloomFilter::findHashWithSeed(const UInt64 & hash, const UInt64 & hash_seed)
+bool BloomFilter::findHashWithSeed(const UInt64 & hash, const UInt64 & hash_seed) const
 {
     size_t pos = fastMod(CityHash_v1_0_2::Hash128to64(CityHash_v1_0_2::uint128(hash, hash_seed)));
     return bool(filter[pos / word_bits] & (1ULL << (pos % word_bits)));

diff --git a/src/Interpreters/BloomFilter.h b/src/Interpreters/BloomFilter.h
@@ -36,19 +36,20 @@ class BloomFilter
     BloomFilter(size_t size_, size_t hashes_, size_t seed_);
 
     void resize(size_t size_);
-    bool find(const char * data, size_t len);
+    bool find(const char * data, size_t len) const;
     void add(const char * data, size_t len);
     void clear();
 
     void addHashWithSeed(const UInt64 & hash, const UInt64 & hash_seed);
-    bool findHashWithSeed(const UInt64 & hash, const UInt64 & hash_seed);
+    bool findHashWithSeed(const UInt64 & hash, const UInt64 & hash_seed) const;
 
     /// Checks if this contains everything from another bloom filter.
     /// Bloom filters must have equal size and seed.
     bool contains(const BloomFilter & bf);
 
     const Container & getFilter() const { return filter; }
     Container & getFilter() { return filter; }
+    size_t getFilterSizeBytes() const { return size; }
 
     /// For debug.
     UInt64 isEmpty() const;

diff --git a/src/Interpreters/GinFilter.h b/src/Interpreters/GinFilter.h
@@ -7,7 +7,7 @@ namespace DB
 {
 
 static constexpr UInt64 DEFAULT_NGRAM_SIZE = 3;
-static constexpr auto DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE = 0.001; /// 0.1%
+static constexpr auto BLOOM_FILTER_FALSE_POSITIVE_RATE = 0.001; /// 0.1%
 
 static inline constexpr auto TEXT_INDEX_NAME = "text";
 

diff --git a/src/Interpreters/ITokenExtractor.cpp b/src/Interpreters/ITokenExtractor.cpp
@@ -280,20 +280,20 @@ void DefaultTokenExtractor::substringToBloomFilter(const char * data, size_t len
             bloom_filter.add(data + token_start, token_len);
 }
 
-void DefaultTokenExtractor::substringToGinFilter(const char * data, size_t length, GinQueryString & query_string, bool is_prefix, bool is_suffix) const
+void DefaultTokenExtractor::substringToTokens(const char * data, size_t length, std::vector<String> & tokens, bool is_prefix, bool is_suffix) const
 {
-    query_string.setQueryString({data, length});
-
     size_t cur = 0;
     size_t token_start = 0;
     size_t token_len = 0;
 
     while (cur < length && nextInString(data, length, &cur, &token_start, &token_len))
+    {
         // In order to avoid filter updates with incomplete tokens,
         // first token is ignored, unless substring is prefix and
         // last token is ignored, unless substring is suffix
         if ((token_start > 0 || is_prefix) && (token_start + token_len < length || is_suffix))
-            query_string.addToken({data + token_start, token_len});
+            tokens.push_back({data + token_start, token_len});
+    }
 }
 
 SplitTokenExtractor::SplitTokenExtractor(const std::vector<String> & separators_)

diff --git a/src/Interpreters/ITokenExtractor.h b/src/Interpreters/ITokenExtractor.h
@@ -12,7 +12,12 @@ namespace DB
 /// Interface for string parsers.
 struct ITokenExtractor
 {
+    ITokenExtractor() = default;
+    ITokenExtractor(const ITokenExtractor &) = default;
+    ITokenExtractor & operator=(const ITokenExtractor &) = default;
+
     virtual ~ITokenExtractor() = default;
+    virtual std::unique_ptr<ITokenExtractor> clone() const = 0;
 
     /// Fast inplace implementation for regular use.
     /// Gets string (data ptr and len) and start position for extracting next token (state of extractor).
@@ -55,29 +60,24 @@ struct ITokenExtractor
 
     virtual void stringLikeToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const = 0;
 
-    /// Updates GIN filter from exact-match string filter value
-    virtual void stringToGinFilter(const char * data, size_t length, GinQueryString & query_string) const = 0;
+    /// Collects copy of tokens into vector. This method is inefficient and should be used only for constants.
+    virtual void stringToTokens(const char * data, size_t length, std::vector<String> & tokens) const = 0;
 
-    /// Updates GIN filter from substring-match string filter value.
+    /// Collects copy of tokens into vector from substring-match string filter value.
     /// An `ITokenExtractor` implementation may decide to skip certain
     /// tokens depending on whether the substring is a prefix or a suffix.
-    virtual void substringToGinFilter(
+    /// This method is inefficient and should be used only for constants.
+    virtual void substringToTokens(
         const char * data,
         size_t length,
-        GinQueryString & query_string,
+        std::vector<String> & tokens,
         bool /*is_prefix*/,
         bool /*is_suffix*/) const
     {
-        stringToGinFilter(data, length, query_string);
-    }
-
-    virtual void stringPaddedToGinFilter(const char * data, size_t length, GinQueryString & query_string) const
-    {
-        stringToGinFilter(data, length, query_string);
+        stringToTokens(data, length, tokens);
     }
 
-    virtual void stringLikeToGinFilter(const char * data, size_t length, GinQueryString & query_string) const = 0;
-
+    virtual void stringLikeToTokens(const char * data, size_t length, std::vector<String> & tokens) const = 0;
     virtual bool supportsStringLike() const = 0;
 };
 
@@ -86,6 +86,11 @@ using TokenExtractorPtr = const ITokenExtractor *;
 template <typename Derived>
 class ITokenExtractorHelper : public ITokenExtractor
 {
+    std::unique_ptr<ITokenExtractor> clone() const override
+    {
+        return std::make_unique<Derived>(*static_cast<const Derived *>(this));
+    }
+
     void stringToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter) const override
     {
         size_t cur = 0;
@@ -115,35 +120,23 @@ class ITokenExtractorHelper : public ITokenExtractor
             bloom_filter.add(token.c_str(), token.size());
     }
 
-    void stringToGinFilter(const char * data, size_t length, GinQueryString & query_string) const override
+    void stringToTokens(const char * data, size_t length, std::vector<String> & tokens) const override
     {
-        query_string.setQueryString({data, length});
-        const auto & tokens = getTokensView(data, length);
-        for (const auto & token : tokens)
-            query_string.addToken(token);
-    }
-
-    void stringPaddedToGinFilter(const char * data, size_t length, GinQueryString & query_string) const override
-    {
-        query_string.setQueryString({data, length});
-
         size_t cur = 0;
         size_t token_start = 0;
         size_t token_len = 0;
 
-        while (cur < length && static_cast<const Derived *>(this)->nextInStringPadded(data, length, &cur, &token_start, &token_len))
-            query_string.addToken({data + token_start, token_len});
+        while (cur < length && static_cast<const Derived *>(this)->nextInString(data, length, &cur, &token_start, &token_len))
+            tokens.push_back({data + token_start, token_len});
     }
 
-    void stringLikeToGinFilter(const char * data, size_t length, GinQueryString & query_string) const override
+    void stringLikeToTokens(const char * data, size_t length, std::vector<String> & tokens) const override
     {
-        query_string.setQueryString({data, length});
-
         size_t cur = 0;
         String token;
 
         while (cur < length && static_cast<const Derived *>(this)->nextInStringLike(data, length, &cur, token))
-            query_string.addToken(token);
+            tokens.push_back(token);
     }
 };
 
@@ -177,7 +170,7 @@ struct DefaultTokenExtractor final : public ITokenExtractorHelper<DefaultTokenEx
     bool nextInStringPadded(const char * data, size_t length, size_t * __restrict pos, size_t * __restrict token_start, size_t * __restrict token_length) const override;
     bool nextInStringLike(const char * data, size_t length, size_t * __restrict pos, String & token) const override;
     void substringToBloomFilter(const char * data, size_t length, BloomFilter & bloom_filter, bool is_prefix, bool is_suffix) const override;
-    void substringToGinFilter(const char * data, size_t length, GinQueryString & query_string, bool is_prefix, bool is_suffix) const override;
+    void substringToTokens(const char * data, size_t length, std::vector<String> & tokens, bool is_prefix, bool is_suffix) const override;
 
     bool supportsStringLike() const override { return true; }
 };

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -809,7 +809,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
 
                 MergeTreeWhereOptimizer where_optimizer{
                     std::move(column_compressed_sizes),
-                    metadata_snapshot,
+                    storage_snapshot,
                     storage->getConditionSelectivityEstimatorByPredicate(storage_snapshot, nullptr, context),
                     queried_columns,
                     supported_prewhere_columns,

diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h
@@ -138,6 +138,7 @@ using Stack = std::vector<Frame>;
 
 /// Second pass optimizations
 void optimizePrimaryKeyConditionAndLimit(const Stack & stack);
+void optimizeDirectReadFromTextIndex(const Stack & stack, QueryPlan::Nodes & nodes);
 void optimizePrewhere(Stack & stack, QueryPlan::Nodes & nodes);
 void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes);
 void optimizeAggregationInOrder(QueryPlan::Node & node, QueryPlan::Nodes &);

diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp
@@ -66,6 +66,9 @@ namespace Setting
     extern const SettingsUInt64 distributed_plan_max_rows_to_broadcast;
     extern const SettingsBool distributed_plan_force_shuffle_aggregation;
     extern const SettingsBool distributed_aggregation_memory_efficient;
+    extern const SettingsBool query_plan_direct_read_from_text_index;
+    extern const SettingsBool use_skip_indexes;
+    extern const SettingsBool use_skip_indexes_on_data_read;
     extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas;
     extern const SettingsNonZeroUInt64 max_parallel_replicas;
 }
@@ -129,6 +132,7 @@ QueryPlanOptimizationSettings::QueryPlanOptimizationSettings(
     optimize_projection = from[Setting::optimize_use_projections];
     use_query_condition_cache = from[Setting::use_query_condition_cache] && from[Setting::allow_experimental_analyzer];
     query_condition_cache_store_conditions_as_plaintext = from[Setting::query_condition_cache_store_conditions_as_plaintext];
+    direct_read_from_text_index = from[Setting::query_plan_direct_read_from_text_index] && from[Setting::use_skip_indexes] && from[Setting::use_skip_indexes_on_data_read] && !use_parallel_replicas;
     query_condition_cache_selectivity_threshold = from[Setting::query_condition_cache_selectivity_threshold];
 
     optimize_use_implicit_projections = optimize_projection && from[Setting::optimize_use_implicit_projections];

diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h
@@ -113,6 +113,9 @@ struct QueryPlanOptimizationSettings
     bool vector_search_with_rescoring;
     VectorSearchFilterStrategy vector_search_filter_strategy;
 
+    /// If full text search using index in payload is enabled.
+    bool direct_read_from_text_index;
+
     /// Setting needed for Sets (JOIN -> IN optimization)
 
     SizeLimits network_transfer_limits;