Skip to content

Commit 57c2c69

Browse files
bharatncbharatnc
committed
Merge 'master' into enable_shared_storage_snapshot_in_query_fix2
2 parents 5528dd1 + 2771c8b commit 57c2c69

File tree

951 files changed

+13269
-12295
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

951 files changed

+13269
-12295
lines changed

‎.clang-tidy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,10 @@ Checks: [
3333
'-bugprone-not-null-terminated-result',
3434
'-bugprone-forward-declaration-namespace',
3535
'-bugprone-nondeterministic-pointer-iteration-order',
36+
'-bugprone-sizeof-expression', # too aggressive
3637

3738
'-cert-dcl16-c',
39+
'-cert-arr39-c', # alias of bugprone-sizeof-expression
3840
'-cert-err58-cpp',
3941

4042
'-clang-analyzer-core.BitwiseShift', # false positive on boost::dynamic_bitset::to_ulong
@@ -43,6 +45,9 @@ Checks: [
4345

4446
'-clang-analyzer-security.ArrayBound', # only false positives
4547

48+
'-clang-analyzer-cplusplus.NewDeleteLeaks', # false positives, slow
49+
'-clang-analyzer-unix.Malloc', # fixme, false positive in contrib somehow
50+
4651
'-cppcoreguidelines-*', # impractical in a codebase as large as ClickHouse, also slow
4752

4853
'-darwin-*',

‎.github/workflows/master.yml‎

Lines changed: 12 additions & 126 deletions
Large diffs are not rendered by default.

‎.github/workflows/nightly_fuzzers.yml‎

Lines changed: 343 additions & 1 deletion
Large diffs are not rendered by default.

‎.github/workflows/pull_request.yml‎

Lines changed: 1 addition & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -625,44 +625,6 @@ jobs:
625625
python3 -m praktika run 'Build (arm_asan)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
626626
fi
627627
628-
build_arm_coverage:
629-
runs-on: [self-hosted, arm-large]
630-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check, fast_test, build_arm_tidy]
631-
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9jb3ZlcmFnZSk=') }}
632-
name: "Build (arm_coverage)"
633-
outputs:
634-
data: ${{ steps.run.outputs.DATA }}
635-
pipeline_status: ${{ steps.run.outputs.pipeline_status }}
636-
steps:
637-
- name: Checkout code
638-
uses: actions/checkout@v4
639-
with:
640-
ref: ${{ env.CHECKOUT_REF }}
641-
642-
- name: Prepare env script
643-
run: |
644-
rm -rf ./ci/tmp
645-
mkdir -p ./ci/tmp
646-
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
647-
export PYTHONPATH=./ci:.:
648-
649-
cat > ./ci/tmp/workflow_status.json << 'EOF'
650-
${{ toJson(needs) }}
651-
EOF
652-
ENV_SETUP_SCRIPT_EOF
653-
654-
- name: Run
655-
id: run
656-
run: |
657-
echo "pipeline_status=undefined" >> $GITHUB_OUTPUT
658-
. ./ci/tmp/praktika_setup_env.sh
659-
set -o pipefail
660-
if command -v ts &> /dev/null; then
661-
python3 -m praktika run 'Build (arm_coverage)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
662-
else
663-
python3 -m praktika run 'Build (arm_coverage)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
664-
fi
665-
666628
build_arm_binary:
667629
runs-on: [self-hosted, arm-large]
668630
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check, fast_test, build_arm_tidy]
@@ -4807,7 +4769,7 @@ jobs:
48074769
48084770
finish_workflow:
48094771
runs-on: [self-hosted, style-checker-aarch64]
4810-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check, pr_formatter, docs_check, fast_test, build_arm_tidy, build_amd_debug, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_asan, build_arm_coverage, build_arm_binary, build_arm_tsan, build_amd_release, build_arm_release, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stateless_tests_amd_asan_distributed_plan_sequential, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_parallel, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_sequential, stateless_tests_amd_binary_parallelreplicas_s3_storage_parallel, stateless_tests_amd_binary_parallelreplicas_s3_storage_sequential, stateless_tests_amd_debug_asyncinsert_s3_storage_parallel, stateless_tests_amd_debug_asyncinsert_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_msan_parallel_1_2, stateless_tests_amd_msan_parallel_2_2, stateless_tests_amd_msan_sequential_1_2, stateless_tests_amd_msan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_tsan_s3_storage_parallel, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, bugfix_validation_integration_tests, bugfix_validation_functional_tests, stateless_tests_amd_asan_flaky_check, integration_tests_amd_asan_old_analyzer_1_6, integration_tests_amd_asan_old_analyzer_2_6, integration_tests_amd_asan_old_analyzer_3_6, integration_tests_amd_asan_old_analyzer_4_6, integration_tests_amd_asan_old_analyzer_5_6, integration_tests_amd_asan_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_amd_asan_flaky, docker_server_image, docker_keeper_image, install_packages_amd_debug, compatibility_check_amd_release, compatibility_check_arm_release, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_amd_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_6, performance_comparison_amd_release_master_head_2_6, performance_comparison_amd_release_master_head_3_6, performance_comparison_amd_release_master_head_4_6, performance_comparison_amd_release_master_head_5_6, performance_comparison_amd_release_master_head_6_6, performance_comparison_arm_release_master_head_1_6, performance_comparison_arm_release_master_head_2_6, performance_comparison_arm_release_master_head_3_6, performance_comparison_arm_release_master_head_4_6, performance_comparison_arm_release_master_head_5_6, performance_comparison_arm_release_master_head_6_6]
4772+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check, pr_formatter, docs_check, fast_test, build_arm_tidy, build_amd_debug, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_asan, build_arm_binary, build_arm_tsan, build_amd_release, build_arm_release, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stateless_tests_amd_asan_distributed_plan_sequential, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_parallel, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_sequential, stateless_tests_amd_binary_parallelreplicas_s3_storage_parallel, stateless_tests_amd_binary_parallelreplicas_s3_storage_sequential, stateless_tests_amd_debug_asyncinsert_s3_storage_parallel, stateless_tests_amd_debug_asyncinsert_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_msan_parallel_1_2, stateless_tests_amd_msan_parallel_2_2, stateless_tests_amd_msan_sequential_1_2, stateless_tests_amd_msan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_tsan_s3_storage_parallel, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, bugfix_validation_integration_tests, bugfix_validation_functional_tests, stateless_tests_amd_asan_flaky_check, integration_tests_amd_asan_old_analyzer_1_6, integration_tests_amd_asan_old_analyzer_2_6, integration_tests_amd_asan_old_analyzer_3_6, integration_tests_amd_asan_old_analyzer_4_6, integration_tests_amd_asan_old_analyzer_5_6, integration_tests_amd_asan_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_amd_asan_flaky, docker_server_image, docker_keeper_image, install_packages_amd_debug, compatibility_check_amd_release, compatibility_check_arm_release, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_amd_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_6, performance_comparison_amd_release_master_head_2_6, performance_comparison_amd_release_master_head_3_6, performance_comparison_amd_release_master_head_4_6, performance_comparison_amd_release_master_head_5_6, performance_comparison_amd_release_master_head_6_6, performance_comparison_arm_release_master_head_1_6, performance_comparison_arm_release_master_head_2_6, performance_comparison_arm_release_master_head_3_6, performance_comparison_arm_release_master_head_4_6, performance_comparison_arm_release_master_head_5_6, performance_comparison_arm_release_master_head_6_6]
48114773
if: ${{ !cancelled() }}
48124774
name: "Finish Workflow"
48134775
outputs:

‎CMakeLists.txt‎

Lines changed: 16 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,20 @@
11
cmake_minimum_required(VERSION 3.25)
22

3+
# Some CMakeLists.txt in contrib/ are not being good citizens and change cmake cache variables
4+
# globally. This policy prevents such *cache* variable assignments from altering *normal* variables
5+
# that we've set.
6+
#
7+
# Example: suppose our root CMakeLists.txt assigns a normal variable:
8+
# set(CMAKE_CXX_STANDARD 23)
9+
# then llvm-project assigns cache variable with the same name:
10+
# set(CMAKE_CXX_STANDARD 17 CACHE STRING "...")
11+
# The latter assignment doesn't affect the normal variable, so llvm-project effectively uses C++23.
12+
#
13+
# Without this policy, the behavior is nonlocal and brittle: if the cache variable doesn't exists,
14+
# the normal variable is unset; otherwise it's left alone. So the effective value would depend e.g.
15+
# on the order in which contrib/ libraries are traversed.
16+
set (CMAKE_POLICY_DEFAULT_CMP0126 NEW)
17+
318
project(ClickHouse LANGUAGES C CXX ASM)
419

520
# If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue.
@@ -44,39 +59,9 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md")
4459
message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init")
4560
endif ()
4661

47-
# Take care to add prlimit in command line before ccache, or else ccache thinks that
48-
# prlimit is compiler, and clang++ is its input file, and refuses to work with
49-
# multiple inputs, e.g in ccache log:
50-
# [2021-03-31T18:06:32.655327 36900] Command line: /usr/bin/ccache prlimit --as=10000000000 --data=5000000000 --cpu=600 /usr/bin/clang++-11 - ...... std=gnu++2a -MD -MT src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -MF src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o.d -o src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -c ../src/Storages/MergeTree/IMergeTreeDataPart.cpp
51-
#
52-
# [2021-03-31T18:06:32.656704 36900] Multiple input files: /usr/bin/clang++-11 and ../src/Storages/MergeTree/IMergeTreeDataPart.cpp
53-
#
54-
# Another way would be to use --ccache-skip option before clang++-11 to make
55-
# ccache ignore it.
5662
option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF)
5763
if (ENABLE_CHECK_HEAVY_BUILDS)
58-
# set DATA (since RSS does not work since 2.6.x+) to 5G
59-
set (RLIMIT_DATA 5000000000)
60-
# set VIRT (RLIMIT_AS) to 10G (DATA*2)
61-
set (RLIMIT_AS 10000000000)
62-
# set CPU time limit to 1000 seconds
63-
set (RLIMIT_CPU 1000)
64-
65-
# Sanitizers are too heavy. Some architectures too.
66-
if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE OR ARCH_RISCV64 OR ARCH_LOONGARCH64)
67-
# Twice as large
68-
set (RLIMIT_DATA 10000000000)
69-
set (RLIMIT_AS 20000000000)
70-
set (RLIMIT_CPU 2000)
71-
endif()
72-
73-
# For some files currently building RISCV64/LOONGARCH64 might be too slow.
74-
# TODO: Improve compilation times per file
75-
if (ARCH_RISCV64 OR ARCH_LOONGARCH64)
76-
set (RLIMIT_CPU 1800)
77-
endif()
78-
79-
set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER})
64+
enable_heavy_build_check_if_needed()
8065
endif ()
8166

8267
if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")

‎base/base/getPageSize.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22
#include <unistd.h>
33
#include <cstdlib>
44

5-
Int64 getPageSize()
5+
Int64 getPageSizeImpl()
66
{
77
Int64 page_size = sysconf(_SC_PAGESIZE);
88
if (page_size < 0)
99
abort();
1010
return page_size;
1111
}
12+
13+
Int64 staticPageSize = getPageSizeImpl();

‎base/base/getPageSize.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
#pragma once
22

3+
#include <base/defines.h>
34
#include <base/types.h>
45

6+
extern Int64 staticPageSize;
57

68
/// Get memory page size
7-
Int64 getPageSize();
9+
Int64 inline ALWAYS_INLINE getPageSize()
10+
{
11+
return staticPageSize;
12+
}

‎base/base/types.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ struct hash<Int8> /// NOLINT (cert-dcl58-cpp)
2222
};
2323
}
2424

25+
using size_t = std::size_t;
26+
2527
using UInt16 = uint16_t;
2628
using UInt32 = uint32_t;
2729
using UInt64 = uint64_t;

‎base/base/wide_integer.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <limits>
2727
#include <type_traits>
2828
#include <initializer_list>
29+
#include <base/types.h>
2930

3031
// NOLINTBEGIN(*)
3132

‎base/poco/NetSSL_OpenSSL/src/SSLManager.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,10 @@ void SSLManager::initDefaultContext(bool server)
246246
params.certificateFile = config.getString(prefix + CFG_CERTIFICATE_FILE, params.privateKeyFile);
247247
params.caLocation = config.getString(prefix + CFG_CA_LOCATION, "");
248248

249-
if (server && params.certificateFile.empty() && params.privateKeyFile.empty())
249+
bool keys_are_explicitly_set = !params.privateKeyFile.empty() && !params.certificateFile.empty();
250+
bool acme_certificate_provided = config.has("acme");
251+
252+
if (server && !keys_are_explicitly_set && !acme_certificate_provided)
250253
throw SSLException("Configuration error: no certificate file has been specified");
251254

252255
// optional options for which we have defaults defined

0 commit comments

Comments
 (0)