Skip to content

Commit 6685c27

Browse files
alexey-milovidovalexey-milovidov
authored
Merge branch 'master' into httcpp
2 parents 97e5932 + 9efbe86 commit 6685c27

File tree

172 files changed

+1126
-3907
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

172 files changed

+1126
-3907
lines changed

‎base/base/defines.h‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,18 +73,6 @@
7373
# endif
7474
#endif
7575

76-
#if defined(ADDRESS_SANITIZER)
77-
# define BOOST_USE_ASAN 1
78-
# define BOOST_USE_UCONTEXT 1
79-
#endif
80-
81-
#if defined(THREAD_SANITIZER)
82-
# define BOOST_USE_TSAN 1
83-
# define BOOST_USE_UCONTEXT 1
84-
#endif
85-
86-
/// TODO: Strange enough, there is no way to detect UB sanitizer.
87-
8876
/// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.
8977
/// It is useful in case when compiler cannot see (and exploit) it, but UBSan can.
9078
/// Example: multiplication of signed integers with possibility of overflow when both sides are from user input.

‎contrib/boost-cmake/CMakeLists.txt‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ add_library (boost::system ALIAS _boost_system)
9292
target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR})
9393

9494
# context
95+
option (BOOST_USE_UCONTEXT "Use ucontext_t for context switching of boost::fiber within boost::context" OFF)
96+
9597
enable_language(ASM)
9698
SET(ASM_OPTIONS "-x assembler-with-cpp")
9799

@@ -100,20 +102,6 @@ set (SRCS_CONTEXT
100102
"${LIBRARY_DIR}/libs/context/src/posix/stack_traits.cpp"
101103
)
102104

103-
if (SANITIZE AND (SANITIZE STREQUAL "address" OR SANITIZE STREQUAL "thread"))
104-
add_compile_definitions(BOOST_USE_UCONTEXT)
105-
106-
if (SANITIZE STREQUAL "address")
107-
add_compile_definitions(BOOST_USE_ASAN)
108-
elseif (SANITIZE STREQUAL "thread")
109-
add_compile_definitions(BOOST_USE_TSAN)
110-
endif()
111-
112-
set (SRCS_CONTEXT ${SRCS_CONTEXT}
113-
"${LIBRARY_DIR}/libs/context/src/fiber.cpp"
114-
"${LIBRARY_DIR}/libs/context/src/continuation.cpp"
115-
)
116-
endif()
117105
if (ARCH_AARCH64)
118106
set (SRCS_CONTEXT ${SRCS_CONTEXT}
119107
"${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_elf_gas.S"
@@ -152,10 +140,27 @@ else()
152140
)
153141
endif()
154142

143+
if (SANITIZE OR BOOST_USE_UCONTEXT)
144+
list (APPEND SRCS_CONTEXT
145+
"${LIBRARY_DIR}/libs/context/src/fiber.cpp"
146+
"${LIBRARY_DIR}/libs/context/src/continuation.cpp"
147+
)
148+
endif()
149+
155150
add_library (_boost_context ${SRCS_CONTEXT})
156151
add_library (boost::context ALIAS _boost_context)
157152
target_include_directories (_boost_context PRIVATE ${LIBRARY_DIR})
158153

154+
if (SANITIZE OR BOOST_USE_UCONTEXT)
155+
target_compile_definitions(_boost_context PUBLIC BOOST_USE_UCONTEXT)
156+
endif()
157+
158+
if (SANITIZE STREQUAL "address")
159+
target_compile_definitions(_boost_context PUBLIC BOOST_USE_ASAN)
160+
elseif (SANITIZE STREQUAL "thread")
161+
target_compile_definitions(_boost_context PUBLIC BOOST_USE_TSAN)
162+
endif()
163+
159164
# coroutine
160165

161166
set (SRCS_COROUTINE

‎docker/test/util/process_functional_tests_result.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,16 @@ def process_test_log(log_path, broken_tests):
5959

6060
total += 1
6161
if TIMEOUT_SIGN in line:
62-
failed += 1
63-
test_results.append((test_name, "Timeout", test_time, []))
62+
if test_name in broken_tests:
63+
success += 1
64+
test_results.append((test_name, "BROKEN", test_time, []))
65+
else:
66+
failed += 1
67+
test_results.append((test_name, "Timeout", test_time, []))
6468
elif FAIL_SIGN in line:
6569
if test_name in broken_tests:
6670
success += 1
67-
test_results.append((test_name, "OK", test_time, []))
71+
test_results.append((test_name, "BROKEN", test_time, []))
6872
else:
6973
failed += 1
7074
test_results.append((test_name, "FAIL", test_time, []))
@@ -76,11 +80,11 @@ def process_test_log(log_path, broken_tests):
7680
test_results.append((test_name, "SKIPPED", test_time, []))
7781
else:
7882
if OK_SIGN in line and test_name in broken_tests:
79-
failed += 1
83+
skipped += 1
8084
test_results.append(
8185
(
8286
test_name,
83-
"SKIPPED",
87+
"NOT_FAILED",
8488
test_time,
8589
["This test passed. Update broken_tests.txt.\n"],
8690
)

‎docs/en/development/architecture.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ A hand-written recursive descent parser parses a query. For example, `ParserSele
9898
9999
## Interpreters {#interpreters}
100100

101-
Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery` and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the INSERT query is the `IBlockOutputStream` to write data for insertion to, and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time.
101+
Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery` and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the `INSERT` query is the `IBlockOutputStream` to write data for insertion to, and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time.
102102

103103
`InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations of query.
104104

‎docs/en/sql-reference/statements/show.md‎

Lines changed: 71 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -105,22 +105,6 @@ Result:
105105

106106
- [CREATE DATABASE](https://clickhouse.com/docs/en/sql-reference/statements/create/database/#query-language-create-database)
107107

108-
## SHOW PROCESSLIST
109-
110-
``` sql
111-
SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format]
112-
```
113-
114-
Outputs the content of the [system.processes](../../operations/system-tables/processes.md#system_tables-processes) table, that contains a list of queries that is being processed at the moment, excepting `SHOW PROCESSLIST` queries.
115-
116-
The `SELECT * FROM system.processes` query returns data about all the current queries.
117-
118-
Tip (execute in the console):
119-
120-
``` bash
121-
$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
122-
```
123-
124108
## SHOW TABLES
125109

126110
Displays a list of tables.
@@ -284,6 +268,77 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2
284268
└──────────────┘
285269
```
286270

271+
## SHOW INDEX
272+
273+
Displays a list of primary and data skipping indexes of a table.
274+
275+
```sql
276+
SHOW [EXTENDED] {INDEX | INDEXES | KEYS } {FROM | IN} <table> [{FROM | IN} <db>] [WHERE <expr>] [INTO OUTFILE <filename>] [FORMAT <format>]
277+
```
278+
279+
The database and table name can be specified in abbreviated form as `<db>.<table>`, i.e. `FROM tab FROM db` and `FROM db.tab` are
280+
equivalent. If no database is specified, the query assumes the current database as database.
281+
282+
The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility.
283+
284+
`SHOW INDEX` produces a result table with the following structure:
285+
- table - The name of the table (String)
286+
- non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8)
287+
- key_name - The name of the index, `PRIMARY` if the index is a primary key index (String)
288+
- seq_in_index - Currently unused
289+
- column_name - Currently unused
290+
- collation - The sorting of the column in the index, `A` if ascending, `D` if descending, `NULL` if unsorted (Nullable(String))
291+
- cardinality - Currently unused
292+
- sub_part - Currently unused
293+
- packed - Currently unused
294+
- null - Currently unused
295+
- index_type - The index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String)
296+
- comment - Currently unused
297+
- index_comment - Currently unused
298+
- visible - If the index is visible to the optimizer, always `YES` (String)
299+
- expression - The index expression (String)
300+
301+
**Examples**
302+
303+
Getting information about all indexes in table 'tbl'
304+
305+
```sql
306+
SHOW INDEX FROM 'tbl'
307+
```
308+
309+
Result:
310+
311+
``` text
312+
┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type───┬─comment─┬─index_comment─┬─visible─┬─expression─┐
313+
│ tbl │ 0 │ blf_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ bloom_filter │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ d, b │
314+
│ tbl │ 0 │ mm1_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ a, c, d │
315+
│ tbl │ 0 │ mm2_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, d, e │
316+
│ tbl │ 0 │ PRIMARY │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ A │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ primary │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, a │
317+
│ tbl │ 0 │ set_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ set │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ e │
318+
└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴──────────────┴─────────┴───────────────┴─────────┴────────────┘
319+
```
320+
321+
**See also**
322+
323+
- [system.tables](../../operations/system-tables/tables.md)
324+
- [system.data_skipping_indices](../../operations/system-tables/data_skipping_indices.md)
325+
326+
## SHOW PROCESSLIST
327+
328+
``` sql
329+
SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format]
330+
```
331+
332+
Outputs the content of the [system.processes](../../operations/system-tables/processes.md#system_tables-processes) table, that contains a list of queries that is being processed at the moment, excepting `SHOW PROCESSLIST` queries.
333+
334+
The `SELECT * FROM system.processes` query returns data about all the current queries.
335+
336+
Tip (execute in the console):
337+
338+
``` bash
339+
$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
340+
```
341+
287342
## SHOW GRANTS
288343

289344
Shows privileges for a user.

‎src/AggregateFunctions/AggregateFunctionNothing.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ namespace DB
1313
{
1414
struct Settings;
1515

16+
namespace ErrorCodes
17+
{
18+
extern const int INCORRECT_DATA;
19+
}
20+
1621

1722
/** Aggregate function that takes arbitrary number of arbitrary arguments and does nothing.
1823
*/
@@ -69,7 +74,8 @@ class AggregateFunctionNothing final : public IAggregateFunctionHelper<Aggregate
6974
{
7075
[[maybe_unused]] char symbol;
7176
readChar(symbol, buf);
72-
assert(symbol == '\0');
77+
if (symbol != '\0')
78+
throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect state of aggregate function 'nothing', it should contain exactly one zero byte.");
7379
}
7480

7581
void insertResultInto(AggregateDataPtr __restrict, IColumn & to, Arena *) const override

‎src/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,8 @@ if (TARGET ch_contrib::qpl)
544544
dbms_target_link_libraries(PUBLIC ch_contrib::qpl)
545545
endif ()
546546

547-
dbms_target_link_libraries(PRIVATE _boost_context)
547+
target_link_libraries(clickhouse_common_io PUBLIC boost::context)
548+
dbms_target_link_libraries(PUBLIC boost::context)
548549

549550
if (ENABLE_NLP)
550551
dbms_target_link_libraries (PUBLIC ch_contrib::stemmer)

‎src/Common/ThreadPool.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,10 @@ void ThreadPoolImpl<Thread>::worker(typename std::list<Thread>::iterator thread_
397397

398398
/// We don't run jobs after `shutdown` is set, but we have to properly dequeue all jobs and finish them.
399399
if (shutdown)
400+
{
401+
job_is_done = true;
400402
continue;
403+
}
401404
}
402405

403406
ALLOW_ALLOCATIONS_IN_SCOPE;

‎src/Common/tests/gtest_thread_pool_schedule_exception.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,28 @@ TEST(ThreadPool, ExceptionFromSchedule)
5151
{
5252
EXPECT_TRUE(check());
5353
}
54+
55+
static bool check2()
56+
{
57+
ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 2);
58+
59+
pool.scheduleOrThrowOnError([&]{ throw std::runtime_error("Hello, world!"); });
60+
pool.scheduleOrThrowOnError([]{});
61+
62+
try
63+
{
64+
pool.wait();
65+
}
66+
catch (const std::runtime_error &)
67+
{
68+
return true;
69+
}
70+
71+
return false;
72+
}
73+
74+
TEST(ThreadPool, ExceptionFromWait)
75+
{
76+
for (size_t i = 0; i < 1000; ++i)
77+
EXPECT_TRUE(check2());
78+
}

‎src/Formats/FormatFactory.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -809,6 +809,14 @@ bool FormatFactory::checkIfOutputFormatPrefersLargeBlocks(const String & name) c
809809
return target.prefers_large_blocks;
810810
}
811811

812+
bool FormatFactory::checkParallelizeOutputAfterReading(const String & name, ContextPtr context) const
813+
{
814+
if (name == "Parquet" && context->getSettingsRef().input_format_parquet_preserve_order)
815+
return false;
816+
817+
return true;
818+
}
819+
812820
void FormatFactory::checkFormatName(const String & name) const
813821
{
814822
auto it = dict.find(name);

0 commit comments

Comments
 (0)