Skip to content

Commit 6f3f202

Browse files
alesapinalesapin
authored
Merge branch 'master' into fix_zero_copy_not_atomic
2 parents a08225d + d3c7054 commit 6f3f202

File tree

121 files changed

+868
-391
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

121 files changed

+868
-391
lines changed

‎CMakeLists.txt‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -170,18 +170,6 @@ else ()
170170
set(NO_WHOLE_ARCHIVE --no-whole-archive)
171171
endif ()
172172

173-
option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
174-
if (OS_DARWIN)
175-
# Disable the curl, azure, senry build on MacOS
176-
set (ENABLE_CURL_BUILD OFF)
177-
endif ()
178-
179-
option(ENABLE_ISAL_LIBRARY "Enable ISA-L library ON by default except on aarch64." ON)
180-
if (ARCH_AARCH64)
181-
# Disable ISA-L libray on aarch64.
182-
set (ENABLE_ISAL_LIBRARY OFF)
183-
endif ()
184-
185173
if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
186174
# Can be lld or ld-lld or lld-13 or /path/to/lld.
187175
if (LINKER_NAME MATCHES "lld")
@@ -399,9 +387,9 @@ else()
399387
endif ()
400388

401389
option (ENABLE_GWP_ASAN "Enable Gwp-Asan" ON)
402-
# We use mmap for allocations more heavily in debug builds,
403-
# but GWP-ASan also wants to use mmap frequently,
404-
# and due to a large number of memory mappings,
390+
# We use mmap for allocations more heavily in debug builds,
391+
# but GWP-ASan also wants to use mmap frequently,
392+
# and due to a large number of memory mappings,
405393
# it does not work together well.
406394
if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
407395
set(ENABLE_GWP_ASAN OFF)

‎contrib/CMakeLists.txt‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -141,20 +141,19 @@ add_contrib (libuv-cmake libuv)
141141
add_contrib (liburing-cmake liburing)
142142
add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
143143
add_contrib (cassandra-cmake cassandra) # requires: libuv
144-
145-
if (ENABLE_CURL_BUILD)
144+
if (NOT OS_DARWIN)
146145
add_contrib (curl-cmake curl)
147146
add_contrib (azure-cmake azure)
148147
add_contrib (sentry-native-cmake sentry-native) # requires: curl
149148
endif()
150-
151149
add_contrib (fmtlib-cmake fmtlib)
152150
add_contrib (krb5-cmake krb5)
153151
add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5
154152
add_contrib (libgsasl-cmake libgsasl) # requires krb5
155153
add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl
156154
add_contrib (nats-io-cmake nats-io)
157-
add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5
155+
add_contrib (isa-l-cmake isa-l)
156+
add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5, isa-l
158157
add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3
159158
add_contrib (cppkafka-cmake cppkafka)
160159
add_contrib (libpqxx-cmake libpqxx)
@@ -178,23 +177,14 @@ add_contrib (s2geometry-cmake s2geometry)
178177
add_contrib (c-ares-cmake c-ares)
179178
add_contrib (qpl-cmake qpl)
180179
add_contrib (morton-nd-cmake morton-nd)
181-
182180
if (ARCH_S390X)
183181
add_contrib(crc32-s390x-cmake crc32-s390x)
184182
endif()
185-
186183
add_contrib (annoy-cmake annoy)
187-
188184
add_contrib (xxHash-cmake xxHash)
189-
190185
add_contrib (google-benchmark-cmake google-benchmark)
191-
192186
add_contrib (ulid-c-cmake ulid-c)
193187

194-
if (ENABLE_ISAL_LIBRARY)
195-
add_contrib (isa-l-cmake isa-l)
196-
endif()
197-
198188
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
199189
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
200190
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,

‎contrib/isa-l-cmake/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
if (NOT ARCH_AMD64)
2+
return()
3+
endif ()
4+
15
set(ISAL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/isa-l")
26

37
# The YASM and NASM assembers are somewhat mutually compatible. ISAL specifically needs NASM. If only YASM is installed, then check_language(ASM_NASM)

‎contrib/libhdfs3-cmake/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ if (TARGET OpenSSL::SSL)
172172
target_link_libraries(_hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL)
173173
endif()
174174

175-
if (ENABLE_ISAL_LIBRARY)
175+
if (TARGET ch_contrib::isal)
176176
target_link_libraries(_hdfs3 PRIVATE ch_contrib::isal)
177177
add_definitions(-DHADOOP_ISAL_LIBRARY)
178178
endif()

‎docker/keeper/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
3232
esac
3333

3434
ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release"
35-
ARG VERSION="23.4.1.1943"
35+
ARG VERSION="23.4.2.11"
3636
ARG PACKAGES="clickhouse-keeper"
3737

3838
# user/group precreated explicitly with fixed uid/gid on purpose.

‎docker/server/Dockerfile.alpine‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
3333
# lts / testing / prestable / etc
3434
ARG REPO_CHANNEL="stable"
3535
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
36-
ARG VERSION="23.4.1.1943"
36+
ARG VERSION="23.4.2.11"
3737
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
3838

3939
# user/group precreated explicitly with fixed uid/gid on purpose.

‎docker/server/Dockerfile.ubuntu‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
2222

2323
ARG REPO_CHANNEL="stable"
2424
ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
25-
ARG VERSION="23.4.1.1943"
25+
ARG VERSION="23.4.2.11"
2626
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
2727

2828
# set non-empty deb_location_url url to create a docker image

‎docs/changelogs/v23.4.2.11-stable.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
---
2+
sidebar_position: 1
3+
sidebar_label: 2023
4+
---
5+
6+
# 2023 Changelog
7+
8+
### ClickHouse release v23.4.2.11-stable (b6442320f9d) FIXME as compared to v23.4.1.1943-stable (3920eb987f7)
9+
10+
#### Bug Fix (user-visible misbehavior in an official stable release)
11+
12+
* Revert "Fix GCS native copy ([#48981](https://github.com/ClickHouse/ClickHouse/issues/48981))" [#49194](https://github.com/ClickHouse/ClickHouse/pull/49194) ([Raúl Marín](https://github.com/Algunenano)).
13+
* Fix race on Outdated parts loading [#49223](https://github.com/ClickHouse/ClickHouse/pull/49223) ([Alexander Tokmakov](https://github.com/tavplubix)).
14+
15+
#### NOT FOR CHANGELOG / INSIGNIFICANT
16+
17+
* Implement status comment [#48468](https://github.com/ClickHouse/ClickHouse/pull/48468) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
18+
* Update curl to 8.0.1 (for CVEs) [#48765](https://github.com/ClickHouse/ClickHouse/pull/48765) ([Boris Kuschel](https://github.com/bkuschel)).
19+
* Fallback auth gh api [#49314](https://github.com/ClickHouse/ClickHouse/pull/49314) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
20+

‎docs/en/engines/table-engines/mergetree-family/mergetree.md‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,50 @@ Syntax: `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions,
439439
- `number_of_hash_functions` — The number of hash functions used in the Bloom filter.
440440
- `random_seed` — The seed for Bloom filter hash functions.
441441

442+
Users can create [UDF](/docs/en/sql-reference/statements/create/function.md) to estimate the parameters set of `ngrambf_v1`. Query statements are as follows:
443+
444+
```sql
445+
CREATE FUNCTION bfEstimateFunctions [ON CLUSTER cluster]
446+
AS
447+
(total_nubmer_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_nubmer_of_all_grams) * log(2));
448+
449+
CREATE FUNCTION bfEstimateBmSize [ON CLUSTER cluster]
450+
AS
451+
(total_nubmer_of_all_grams, probability_of_false_positives) -> ceil((total_nubmer_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2))));
452+
453+
CREATE FUNCTION bfEstimateFalsePositive [ON CLUSTER cluster]
454+
AS
455+
(total_nubmer_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_nubmer_of_all_grams)), number_of_hash_functions);
456+
457+
CREATE FUNCTION bfEstimateGramNumber [ON CLUSTER cluster]
458+
AS
459+
(number_of_hash_functions, probability_of_false_positives, size_of_bloom_filter_in_bytes) -> ceil(size_of_bloom_filter_in_bytes / (-number_of_hash_functions / log(1 - exp(log(probability_of_false_positives) / number_of_hash_functions))))
460+
461+
```
462+
To use those functions,we need to specify two parameter at least.
463+
For example, if there 4300 ngrams in the granule and we expect false positives to be less than 0.0001. The other parameters can be estimated by executing following queries:
464+
465+
466+
```sql
467+
--- estimate number of bits in the filter
468+
SELECT bfEstimateBmSize(4300, 0.0001) / 8 as size_of_bloom_filter_in_bytes;
469+
470+
┌─size_of_bloom_filter_in_bytes─┐
471+
│ 10304 │
472+
└───────────────────────────────┘
473+
474+
--- estimate number of hash functions
475+
SELECT bfEstimateFunctions(4300, bfEstimateBmSize(4300, 0.0001)) as number_of_hash_functions
476+
477+
┌─number_of_hash_functions─┐
478+
│ 13 │
479+
└──────────────────────────┘
480+
481+
```
482+
Of course, you can also use those functions to estimate parameters by other conditions.
483+
The functions refer to the content [here](https://hur.st/bloomfilter).
484+
485+
442486
#### Token Bloom Filter
443487

444488
The same as `ngrambf_v1`, but stores tokens instead of ngrams. Tokens are sequences separated by non-alphanumeric characters.

‎src/AggregateFunctions/AggregateFunctionMinMaxAny.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -554,7 +554,8 @@ struct SingleValueDataString
554554
if (capacity < size_to_reserve)
555555
{
556556
if (unlikely(MAX_STRING_SIZE < size_to_reserve))
557-
throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "String size is too big ({})", size_to_reserve);
557+
throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "String size is too big ({}), maximum: {}",
558+
size_to_reserve, MAX_STRING_SIZE);
558559

559560
size_t rounded_capacity = roundUpToPowerOfTwoOrZero(size_to_reserve);
560561
chassert(rounded_capacity <= MAX_STRING_SIZE + 1); /// rounded_capacity <= 2^31
@@ -624,7 +625,8 @@ struct SingleValueDataString
624625
void changeImpl(StringRef value, Arena * arena)
625626
{
626627
if (unlikely(MAX_STRING_SIZE < value.size))
627-
throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "String size is too big ({})", value.size);
628+
throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "String size is too big ({}), maximum: {}",
629+
value.size, MAX_STRING_SIZE);
628630

629631
UInt32 value_size = static_cast<UInt32>(value.size);
630632

0 commit comments

Comments
 (0)