Skip to content

Commit 1ee4912

Browse files
authored
Merge pull request #119350 from ClickHouse/pr-table-function-anchor
Read time series table functions on parallel replicas
2 parents aef2647 + 2025bbe commit 1ee4912

28 files changed

Lines changed: 799 additions & 28 deletions

‎src/Analyzer/TableFunctionNode.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
#include <Storages/IStorage.h>
1111
#include <Storages/StorageView.h>
1212

13+
#include <TableFunctions/ITableFunction.h>
14+
1315
#include <Parsers/ASTFunction.h>
1416
#include <Parsers/ASTSetQuery.h>
1517

@@ -53,6 +55,9 @@ void TableFunctionNode::resolve(TableFunctionPtr table_function_value, StoragePt
5355
table_function = std::move(table_function_value);
5456
storage = std::move(storage_value);
5557
storage_id = storage->getStorageID();
58+
/// A parameterized view is resolved as a table function node but has no `ITableFunction` object
59+
/// (see `QueryAnalyzer::resolveTableFunction`), and it cannot be the coordinated read itself.
60+
parallel_replicas_read_id = table_function ? table_function->getParallelReplicasReadID() : std::nullopt;
5661
unresolved_arguments_indexes = std::move(unresolved_arguments_indexes_);
5762

5863
const auto metadata_snapshot = storage->getInMemoryMetadataPtr(context, false);
@@ -177,6 +182,7 @@ QueryTreeNodePtr TableFunctionNode::cloneImpl() const
177182

178183
result->storage = storage;
179184
result->storage_id = storage_id;
185+
result->parallel_replicas_read_id = parallel_replicas_read_id;
180186
result->storage_snapshot = storage_snapshot;
181187
result->table_expression_modifiers = table_expression_modifiers;
182188
result->settings_changes = settings_changes;

‎src/Analyzer/TableFunctionNode.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@ class TableFunctionNode : public ITableExpressionNode
113113
/// Get storage snapshot, throws exception if function node is not resolved
114114
const StorageSnapshotPtr & getStorageSnapshot() const;
115115

116+
/// See `ITableFunction::getParallelReplicasReadID`. Resolved once and stored, rather than asked of
117+
/// `table_function`, because a cloned node does not carry the `ITableFunction` object.
118+
const std::optional<String> & getParallelReplicasReadID() const { return parallel_replicas_read_id; }
119+
116120
const VectorWithMemoryTracking<size_t> & getUnresolvedArgumentIndexes() const
117121
{
118122
return unresolved_arguments_indexes;
@@ -172,6 +176,7 @@ class TableFunctionNode : public ITableExpressionNode
172176
TableFunctionPtr table_function;
173177
StoragePtr storage;
174178
StorageID storage_id;
179+
std::optional<String> parallel_replicas_read_id;
175180
StorageSnapshotPtr storage_snapshot;
176181
VectorWithMemoryTracking<size_t> unresolved_arguments_indexes;
177182
std::optional<TableExpressionModifiers> table_expression_modifiers;

‎src/Interpreters/ClusterProxy/executeQuery.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1185,7 +1185,10 @@ void executeQueryWithParallelReplicas(
11851185
if (new_context->getSettingsRef()[Setting::serialize_query_plan])
11861186
{
11871187
remote_query_plan = createRemotePlanForParallelReplicas(query_tree, *header, new_context, processed_stage);
1188-
remote_query_plan->ensureSerialized(DBMS_QUERY_PLAN_SERIALIZATION_VERSION);
1188+
/// A null plan means the coordinated read could not be marked in it; fall back to sending the
1189+
/// query text, which every replica plans for itself (see `createRemotePlanForParallelReplicas`).
1190+
if (remote_query_plan)
1191+
remote_query_plan->ensureSerialized(DBMS_QUERY_PLAN_SERIALIZATION_VERSION);
11891192
}
11901193

11911194
/// The subquery carries its own SETTINGS (shipped to remote replicas via the AST). Pass its

‎src/Planner/PlannerContext.h‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ namespace DB
1818
*/
1919

2020
class QueryNode;
21-
class TableNode;
21+
class ITableExpressionNode;
2222
class UnionNode;
2323

2424
struct FiltersForTableExpression
@@ -39,7 +39,7 @@ class GlobalPlannerContext
3939
public:
4040
GlobalPlannerContext(
4141
const QueryNode * parallel_replicas_node_,
42-
const TableNode * parallel_replicas_table_,
42+
const ITableExpressionNode * parallel_replicas_table_,
4343
const UnionNode * parallel_replicas_table_union_,
4444
FiltersForTableExpressionMap filters_for_table_expressions_)
4545
: parallel_replicas_node(parallel_replicas_node_)
@@ -80,7 +80,8 @@ class GlobalPlannerContext
8080
const QueryNode * const parallel_replicas_node = nullptr;
8181
/// Table which is used with parallel replicas reading.
8282
/// It is the left-most table of the query (in JOINs, UNIONs and subqueries).
83-
const TableNode * const parallel_replicas_table = nullptr;
83+
/// It's either a table, or a table function that is only a reference to a table.
84+
const ITableExpressionNode * const parallel_replicas_table = nullptr;
8485
/// UNION node whose every child query reads from a table eligible for parallel replicas.
8586
/// When set, each branch retains parallel replicas reading instead of having it disabled.
8687
const UnionNode * const parallel_replicas_table_union = nullptr;

‎src/Planner/PlannerJoinTree.cpp‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1735,8 +1735,13 @@ bool parallelReplicasEnabledForStorage(const StoragePtr & current_storage, const
17351735
/// that a single leaf (the leftmost) takes the WithMergeableState path with the
17361736
/// whole join query tree; the other leaves must take the plain `storage->read`
17371737
/// path. Callers should compute this once for the join tree and gate the
1738-
/// per-leaf parallel-replicas activation accordingly.
1739-
bool allowParallelReplicasForJoinTree(const QueryTreeNodePtr & join_tree_node, const ContextPtr & context, const Settings & query_settings)
1738+
/// per-leaf parallel-replicas activation accordingly. `query_sets_additional_table_filters` says
1739+
/// whether the query sent to the replicas sets `additional_table_filters` anywhere.
1740+
bool allowParallelReplicasForJoinTree(
1741+
const QueryTreeNodePtr & join_tree_node,
1742+
const ContextPtr & context,
1743+
const Settings & query_settings,
1744+
bool query_sets_additional_table_filters)
17401745
{
17411746
if (!join_tree_node)
17421747
return false;
@@ -1763,7 +1768,8 @@ bool allowParallelReplicasForJoinTree(const QueryTreeNodePtr & join_tree_node, c
17631768

17641769
const auto * left_table_function = left_table_expr->as<TableFunctionNode>();
17651770
if (left_table_function)
1766-
return parallelReplicasEnabledForStorage(left_table_function->getStorage(), context, query_settings);
1771+
return tableFunctionCanDriveParallelReplicasRead(*left_table_function, query_sets_additional_table_filters)
1772+
&& parallelReplicasEnabledForStorage(left_table_function->getStorage(), context, query_settings);
17671773

17681774
// check if left one is not subquery
17691775
return left_table_expr->getNodeType() != QueryTreeNodeType::QUERY
@@ -2841,22 +2847,25 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_
28412847
/// It is just a safety check needed until we have a proper sending plan to replicas.
28422848
/// If we have a non-trivial storage like View it might create its own Planner inside read(), run findTableForParallelReplicas()
28432849
/// and find some other table that might be used for reading with parallel replicas. It will lead to errors.
2844-
/// The chosen table and union children are TableNodes, so a table function matches
2845-
/// neither and equality against them is meaningless when table_node is null.
2850+
/// The chosen table is a `TableNode`, or a `TableFunctionNode` that only references a
2851+
/// table, so compare against whichever of the two this table expression is.
2852+
const ITableExpressionNode * table_expression_node
2853+
= table_node ? static_cast<const ITableExpressionNode *>(table_node) : table_function_node;
28462854
const bool no_tables_or_another_table_chosen_for_reading_with_parallel_replicas_mode
28472855
= query_context->canUseParallelReplicasOnFollower()
2848-
&& (!table_node || table_node != planner_context->getGlobalPlannerContext()->parallel_replicas_table);
2856+
&& (!table_expression_node
2857+
|| table_expression_node != planner_context->getGlobalPlannerContext()->parallel_replicas_table);
28492858
if (no_tables_or_another_table_chosen_for_reading_with_parallel_replicas_mode)
28502859
{
28512860
bool disable_parallel_replicas_for_storage = true;
28522861
ContextPtr updated_context = effective_context;
28532862
if (const UnionNode * table_union
2854-
= table_node ? planner_context->getGlobalPlannerContext()->parallel_replicas_table_union : nullptr)
2863+
= table_expression_node ? planner_context->getGlobalPlannerContext()->parallel_replicas_table_union : nullptr)
28552864
{
28562865
SelectQueryOptions options;
28572866
for (const auto & child : table_union->getQueries().getNodes())
28582867
{
2859-
if (table_node == findTableForParallelReplicas(child, options))
2868+
if (table_expression_node == findTableForParallelReplicas(child, options))
28602869
{
28612870
disable_parallel_replicas_for_storage = false;
28622871
break;
@@ -2946,9 +2955,18 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_
29462955
query_plan = std::move(query_plan_parallel_replicas);
29472956
}
29482957
}
2958+
/// A table function drives the read only when it is a stable reference to a table: the
2959+
/// query shipped to the replicas contains the call, and each replica must resolve it to its
2960+
/// own copy of the same table. A table function that builds a storage of its own resolves to
2961+
/// something local to this server, so it stays on a single replica even when that storage
2962+
/// happens to be a `MergeTree`.
29492963
else if (
2964+
const bool query_sets_additional_table_filters
2965+
= queryTreeSetsAdditionalTableFilters(table_expression_query_info.query_tree.get());
29502966
ClusterProxy::canUseParallelReplicasOnInitiator(query_context)
2951-
&& allowParallelReplicasForJoinTree(parent_join_tree, query_context, settings))
2967+
&& (!table_function_node || tableFunctionCanDriveParallelReplicasRead(*table_function_node, query_sets_additional_table_filters))
2968+
&& allowParallelReplicasForJoinTree(
2969+
parent_join_tree, query_context, settings, query_sets_additional_table_filters))
29522970
{
29532971
// (1) find read step
29542972

@@ -3030,6 +3048,11 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(TableExpressionNodePtr table_
30303048
/// plan. Deciding whether to use parallel replicas and where to place the
30313049
/// local/remote boundary is done later, as an analysis of the whole plan
30323050
/// (QueryPlanOptimizations::applyParallelReplicas), which inserts the split step.
3051+
///
3052+
/// For a read through a table function the fragment names the storage the call
3053+
/// resolved to here. The inner table of a `TimeSeries` table is named after the
3054+
/// table's UUID, so a replica that created its table on its own, with another UUID,
3055+
/// fails to find it rather than reading something else.
30333056
QueryPlan query_plan_parallel_replicas;
30343057
storage->read(
30353058
query_plan_parallel_replicas,

‎src/Planner/findParallelReplicasQuery.cpp‎

Lines changed: 90 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include <Analyzer/InDepthQueryTreeVisitor.h>
44
#include <Analyzer/JoinNode.h>
55
#include <Analyzer/QueryNode.h>
6+
#include <Analyzer/TableFunctionNode.h>
67
#include <Analyzer/TableNode.h>
78
#include <Analyzer/UnionNode.h>
89
#include <Core/Settings.h>
@@ -30,6 +31,7 @@ namespace DB
3031
{
3132
namespace Setting
3233
{
34+
extern const SettingsMap additional_table_filters;
3335
extern const SettingsBool parallel_replicas_allow_in_with_subquery;
3436
extern const SettingsBool parallel_replicas_for_non_replicated_merge_tree;
3537
extern const SettingsBool parallel_replicas_for_queries_with_multiple_tables;
@@ -45,7 +47,7 @@ namespace ErrorCodes
4547
extern const int UNSUPPORTED_METHOD;
4648
}
4749

48-
bool isTableNodeEligibleForParallelReplicas(const TableNode & table_node, const StoragePtr & storage, const ContextPtr & context)
50+
static bool isStorageEligibleForParallelReplicas(const StoragePtr & storage, bool has_final, const ContextPtr & context)
4951
{
5052
const auto & settings = context->getSettingsRef();
5153

@@ -60,12 +62,70 @@ bool isTableNodeEligibleForParallelReplicas(const TableNode & table_node, const
6062
return false;
6163

6264
/// Parallel replicas not supported with FINAL.
63-
if (table_node.hasTableExpressionModifiers() && table_node.getTableExpressionModifiers()->hasFinal())
65+
if (has_final)
6466
return false;
6567

6668
return true;
6769
}
6870

71+
bool isTableNodeEligibleForParallelReplicas(const TableNode & table_node, const StoragePtr & storage, const ContextPtr & context)
72+
{
73+
const bool has_final = table_node.hasTableExpressionModifiers() && table_node.getTableExpressionModifiers()->hasFinal();
74+
return isStorageEligibleForParallelReplicas(storage, has_final, context);
75+
}
76+
77+
/// A table function that denotes the same source on every replica (`ITableFunction::getParallelReplicasReadID`)
78+
/// can anchor a parallel-replicas read just like a `TableNode`: every replica resolves the call locally and
79+
/// reads its own copy. A table function that builds a storage of its own cannot - it is not the same table on
80+
/// every replica - so it is not enough for the resolved storage to look like a `MergeTree`.
81+
bool queryTreeSetsAdditionalTableFilters(const IQueryTreeNode * query_tree_node)
82+
{
83+
std::vector<const IQueryTreeNode *> nodes{query_tree_node};
84+
while (!nodes.empty())
85+
{
86+
const auto * node = nodes.back();
87+
nodes.pop_back();
88+
89+
ContextPtr node_context;
90+
if (const auto * query_node = node->as<QueryNode>())
91+
node_context = query_node->getContext();
92+
else if (const auto * union_node = node->as<UnionNode>())
93+
node_context = union_node->getContext();
94+
if (node_context && !node_context->getSettingsRef()[Setting::additional_table_filters].value.empty())
95+
return true;
96+
97+
for (const auto & child : node->getChildren())
98+
{
99+
if (child)
100+
nodes.push_back(child.get());
101+
}
102+
}
103+
return false;
104+
}
105+
106+
bool tableFunctionCanDriveParallelReplicasRead(const TableFunctionNode & table_function_node, bool query_sets_additional_table_filters)
107+
{
108+
if (!table_function_node.getParallelReplicasReadID())
109+
return false;
110+
111+
/// A read through a table function is sent to the replicas as query text even with `serialize_query_plan`,
112+
/// and a replica matches `additional_table_filters` against its own current database, which is not the
113+
/// initiator's. The filters could match here and not there, so the combination is not distributed. The
114+
/// text is the whole query, so filters set in any of its subqueries count, not only around this read.
115+
return !query_sets_additional_table_filters;
116+
}
117+
118+
static bool canUseTableFunctionForParallelReplicas(
119+
const TableFunctionNode & table_function_node, const IQueryTreeNode * query_tree_root, const ContextPtr & context)
120+
{
121+
if (!tableFunctionCanDriveParallelReplicasRead(table_function_node, queryTreeSetsAdditionalTableFilters(query_tree_root)))
122+
return false;
123+
124+
const bool has_final
125+
= table_function_node.hasTableExpressionModifiers() && table_function_node.getTableExpressionModifiers()->hasFinal();
126+
return isStorageEligibleForParallelReplicas(table_function_node.getStorage(), has_final, context);
127+
}
128+
69129
bool canUseTableForParallelReplicas(const TableNode & table_node, const ContextPtr & context)
70130
{
71131
const auto & settings = context->getSettingsRef();
@@ -110,6 +170,7 @@ bool canUseTableForParallelReplicas(const TableNode & table_node, const ContextP
110170
static std::vector<const QueryNode *> getSupportingParallelReplicasQueries(const IQueryTreeNode * query_tree_node, const ContextPtr & context)
111171
{
112172
const auto & settings = context->getSettingsRef();
173+
const IQueryTreeNode * const query_tree_root = query_tree_node;
113174

114175
std::vector<const QueryNode *> res;
115176

@@ -132,6 +193,13 @@ static std::vector<const QueryNode *> getSupportingParallelReplicasQueries(const
132193
}
133194
case QueryTreeNodeType::TABLE_FUNCTION:
134195
{
196+
/// Kept in step with the `TABLE_FUNCTION` case of `findTableForParallelReplicas` below: a node
197+
/// that can anchor the read there must be able to reach this traversal too, otherwise the
198+
/// initiator picks an anchor for a query this one has already refused to distribute.
199+
const auto & table_function_node = query_tree_node->as<TableFunctionNode &>();
200+
if (canUseTableFunctionForParallelReplicas(table_function_node, query_tree_root, current_context))
201+
return res;
202+
135203
return {};
136204
}
137205
case QueryTreeNodeType::QUERY:
@@ -544,9 +612,10 @@ const QueryNode * findQueryForParallelReplicas(const QueryTreeNodePtr & query_tr
544612
return res;
545613
}
546614

547-
static const TableNode * findTableForParallelReplicas(const IQueryTreeNode * query_tree_node, const ContextPtr & context)
615+
static const ITableExpressionNode * findTableForParallelReplicas(const IQueryTreeNode * query_tree_node, const ContextPtr & context)
548616
{
549617
const auto & settings = context->getSettingsRef();
618+
const IQueryTreeNode * const query_tree_root = query_tree_node;
550619
std::stack<const IQueryTreeNode *> join_nodes;
551620
/// Enclosing (sub)query context per pending join branch: must match the one
552621
/// getSupportingParallelReplicasQueries used, or no eligible table is found here.
@@ -577,6 +646,10 @@ static const TableNode * findTableForParallelReplicas(const IQueryTreeNode * que
577646
}
578647
case QueryTreeNodeType::TABLE_FUNCTION:
579648
{
649+
const auto & table_function_node = query_tree_node->as<TableFunctionNode &>();
650+
if (canUseTableFunctionForParallelReplicas(table_function_node, query_tree_root, current_context))
651+
return &table_function_node;
652+
580653
query_tree_node = nullptr;
581654
break;
582655
}
@@ -652,7 +725,17 @@ static const TableNode * findTableForParallelReplicas(const IQueryTreeNode * que
652725
return nullptr;
653726
}
654727

655-
const TableNode * findTableForParallelReplicas(const QueryTreeNodePtr & query_tree_node, const SelectQueryOptions & select_query_options)
728+
StorageID getStorageIDForParallelReplicas(const ITableExpressionNode & table_expression_node)
729+
{
730+
if (const auto * table_node = table_expression_node.as<TableNode>())
731+
return table_node->getStorageID();
732+
733+
const auto * table_function_node = table_expression_node.as<TableFunctionNode>();
734+
chassert(table_function_node);
735+
return table_function_node->getStorageID();
736+
}
737+
738+
const ITableExpressionNode * findTableForParallelReplicas(const QueryTreeNodePtr & query_tree_node, const SelectQueryOptions & select_query_options)
656739
{
657740
if (select_query_options.only_analyze)
658741
return nullptr;
@@ -780,14 +863,14 @@ JoinTreeQueryPlan buildQueryPlanForParallelReplicas(
780863
removeGroupingFunctionSpecializations(modified_query_tree_for_ast);
781864
ASTPtr modified_query_ast = queryNodeToDistributedSelectQuery(modified_query_tree_for_ast);
782865

783-
const TableNode * table_node = findTableForParallelReplicas(modified_query_tree.get(), context);
784-
if (!table_node)
866+
const auto * table_expression_node = findTableForParallelReplicas(modified_query_tree.get(), context);
867+
if (!table_expression_node)
785868
throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't determine table for parallel replicas");
786869

787870
QueryPlan query_plan;
788871
ClusterProxy::executeQueryWithParallelReplicas(
789872
query_plan,
790-
table_node->getStorageID(),
873+
getStorageIDForParallelReplicas(*table_expression_node),
791874
header,
792875
processed_stage,
793876
modified_query_ast,

0 commit comments

Comments
 (0)