Skip to content

Commit 47b5d13

Browse files
alexey-milovidovalexey-milovidov
authored
Merge branch 'master' into fix-stress-test
2 parents acb4326 + be55bbf commit 47b5d13

File tree

208 files changed

+2080
-6506
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

208 files changed

+2080
-6506
lines changed

‎CMakeLists.txt‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,9 +208,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS
208208
"Do not generate debugger info for heavy modules (ClickHouse functions and dictionaries, some contrib)"
209209
${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT})
210210

211-
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
212-
set(USE_DEBUG_HELPERS ON)
213-
endif()
214211
option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
215212

216213
option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF)

‎docker/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
## ClickHouse Dockerfiles
22

3-
This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They are updated in each release.
3+
This directory contain Dockerfiles for `clickhouse-server`. They are updated in each release.
44

55
Also there is bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`.

‎docker/client/Dockerfile‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎docker/client/README.md‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎docker/packager/binary/Dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
101101
python3-boto3 \
102102
yasm \
103103
zstd \
104-
jq \
105104
&& apt-get clean \
106105
&& rm -rf /var/lib/apt/lists
107106

‎docker/packager/binary/build.sh‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ if [ "$BUILD_MUSL_KEEPER" == "1" ]
5959
then
6060
# build keeper with musl separately
6161
# and without rust bindings
62-
cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
62+
cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
6363
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
6464
ninja $NINJA_FLAGS clickhouse-keeper
6565

@@ -74,10 +74,10 @@ then
7474
rm -f CMakeCache.txt
7575

7676
# Build the rest of binaries
77-
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
77+
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
7878
else
7979
# Build everything
80-
cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
80+
cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
8181
fi
8282

8383
# No quotes because I want it to expand to nothing if empty.
@@ -181,11 +181,4 @@ then
181181
tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
182182
fi
183183

184-
# Prepare profile info (time-trace)
185-
mkdir -p profile-tmp
186-
../utils/prepare-time-trace/prepare-time-trace.sh . profile-tmp
187-
find profile-tmp -type f -print0 | xargs -0 cat > /profile/profile.json
188-
189-
wc -c /profile/profile.json
190-
191184
ls -l /output

‎docker/packager/packager‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ def check_image_exists_locally(image_name: str) -> bool:
2222
output = subprocess.check_output(
2323
f"docker images -q {image_name} 2> /dev/null", shell=True
2424
)
25-
return output != ""
25+
return output != b""
2626
except subprocess.CalledProcessError:
2727
return False
2828

@@ -46,7 +46,7 @@ def build_image(image_name: str, filepath: Path) -> None:
4646
)
4747

4848

49-
def pre_build(repo_path: Path, env_variables: List[str]):
49+
def pre_build(repo_path: Path, env_variables: List[str]) -> None:
5050
if "WITH_PERFORMANCE=1" in env_variables:
5151
current_branch = subprocess.check_output(
5252
"git branch --show-current", shell=True, encoding="utf-8"
@@ -78,13 +78,11 @@ def run_docker_image_with_env(
7878
image_name: str,
7979
as_root: bool,
8080
output_dir: Path,
81-
profile_dir: Path,
8281
env_variables: List[str],
8382
ch_root: Path,
8483
ccache_dir: Optional[Path],
85-
):
84+
) -> None:
8685
output_dir.mkdir(parents=True, exist_ok=True)
87-
profile_dir.mkdir(parents=True, exist_ok=True)
8886

8987
env_part = " -e ".join(env_variables)
9088
if env_part:
@@ -106,7 +104,7 @@ def run_docker_image_with_env(
106104

107105
cmd = (
108106
f"docker run --network=host --user={user} --rm {ccache_mount}"
109-
f"--volume={output_dir}:/output --volume={ch_root}:/build --volume={profile_dir}:/profile {env_part} "
107+
f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
110108
f"{interactive} {image_name}"
111109
)
112110

@@ -132,9 +130,10 @@ def parse_env_variables(
132130
version: str,
133131
official: bool,
134132
additional_pkgs: bool,
133+
with_profiler: bool,
135134
with_coverage: bool,
136135
with_binaries: str,
137-
):
136+
) -> List[str]:
138137
DARWIN_SUFFIX = "-darwin"
139138
DARWIN_ARM_SUFFIX = "-darwin-aarch64"
140139
ARM_SUFFIX = "-aarch64"
@@ -325,6 +324,9 @@ def parse_env_variables(
325324
# utils are not included into clickhouse-bundle, so build everything
326325
build_target = "all"
327326

327+
if with_profiler:
328+
cmake_flags.append("-DENABLE_BUILD_PROFILING=1")
329+
328330
if with_coverage:
329331
cmake_flags.append("-DWITH_COVERAGE=1")
330332

@@ -364,7 +366,6 @@ def parse_args() -> argparse.Namespace:
364366
help="ClickHouse git repository",
365367
)
366368
parser.add_argument("--output-dir", type=dir_name, required=True)
367-
parser.add_argument("--profile-dir", type=dir_name, required=True)
368369
parser.add_argument("--debug-build", action="store_true")
369370

370371
parser.add_argument(
@@ -420,6 +421,7 @@ def parse_args() -> argparse.Namespace:
420421
parser.add_argument("--version")
421422
parser.add_argument("--official", action="store_true")
422423
parser.add_argument("--additional-pkgs", action="store_true")
424+
parser.add_argument("--with-profiler", action="store_true")
423425
parser.add_argument("--with-coverage", action="store_true")
424426
parser.add_argument(
425427
"--with-binaries", choices=("programs", "tests", ""), default=""
@@ -455,7 +457,7 @@ def parse_args() -> argparse.Namespace:
455457
return args
456458

457459

458-
def main():
460+
def main() -> None:
459461
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
460462
args = parse_args()
461463

@@ -483,6 +485,7 @@ def main():
483485
args.version,
484486
args.official,
485487
args.additional_pkgs,
488+
args.with_profiler,
486489
args.with_coverage,
487490
args.with_binaries,
488491
)
@@ -492,7 +495,6 @@ def main():
492495
image_with_version,
493496
args.as_root,
494497
args.output_dir,
495-
args.profile_dir,
496498
env_prepared,
497499
ch_root,
498500
args.ccache_dir,

‎docker/test/performance-comparison/compare.sh‎

Lines changed: 21 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -665,9 +665,8 @@ create view partial_query_times as select * from
665665
-- Report for backward-incompatible ('partial') queries that we could only run on the new server (e.g.
666666
-- queries with new functions added in the tested PR).
667667
create table partial_queries_report engine File(TSV, 'report/partial-queries-report.tsv')
668-
settings output_format_decimal_trailing_zeros = 1
669-
as select toDecimal64(time_median, 3) time,
670-
toDecimal64(time_stddev / time_median, 3) relative_time_stddev,
668+
as select round(time_median, 3) time,
669+
round(time_stddev / time_median, 3) relative_time_stddev,
671670
test, query_index, query_display_name
672671
from partial_query_times
673672
join query_display_names using (test, query_index)
@@ -739,28 +738,26 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv')
739738
;
740739
741740
create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv')
742-
settings output_format_decimal_trailing_zeros = 1
743741
as with
744742
-- server_time is sometimes reported as zero (if it's less than 1 ms),
745743
-- so we have to work around this to not get an error about conversion
746744
-- of NaN to decimal.
747745
(left > right ? left / right : right / left) as times_change_float,
748746
isFinite(times_change_float) as times_change_finite,
749-
toDecimal64(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
747+
round(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
750748
times_change_finite
751749
? (left > right ? '-' : '+') || toString(times_change_decimal) || 'x'
752750
: '--' as times_change_str
753751
select
754-
toDecimal64(left, 3), toDecimal64(right, 3), times_change_str,
755-
toDecimal64(diff, 3), toDecimal64(stat_threshold, 3),
752+
round(left, 3), round(right, 3), times_change_str,
753+
round(diff, 3), round(stat_threshold, 3),
756754
changed_fail, test, query_index, query_display_name
757755
from queries where changed_show order by abs(diff) desc;
758756
759757
create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv')
760-
settings output_format_decimal_trailing_zeros = 1
761758
as select
762-
toDecimal64(left, 3), toDecimal64(right, 3), toDecimal64(diff, 3),
763-
toDecimal64(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
759+
round(left, 3), round(right, 3), round(diff, 3),
760+
round(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
764761
from queries where unstable_show order by stat_threshold desc;
765762
766763
@@ -789,11 +786,10 @@ create view total_speedup as
789786
;
790787
791788
create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv')
792-
settings output_format_decimal_trailing_zeros = 1
793789
as with
794790
(times_speedup >= 1
795-
? '-' || toString(toDecimal64(times_speedup, 3)) || 'x'
796-
: '+' || toString(toDecimal64(1 / times_speedup, 3)) || 'x')
791+
? '-' || toString(round(times_speedup, 3)) || 'x'
792+
: '+' || toString(round(1 / times_speedup, 3)) || 'x')
797793
as times_speedup_str
798794
select test, times_speedup_str, queries, bad, changed, unstable
799795
-- Not sure what's the precedence of UNION ALL vs WHERE & ORDER BY, hence all
@@ -817,11 +813,10 @@ create view total_client_time_per_query as select *
817813
'test text, query_index int, client float, server float');
818814
819815
create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv')
820-
settings output_format_decimal_trailing_zeros = 1
821-
as select client, server, toDecimal64(client/server, 3) p,
816+
as select client, server, round(client/server, 3) p,
822817
test, query_display_name
823818
from total_client_time_per_query left join query_display_names using (test, query_index)
824-
where p > toDecimal64(1.02, 3) order by p desc;
819+
where p > round(1.02, 3) order by p desc;
825820
826821
create table wall_clock_time_per_test engine Memory as select *
827822
from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float');
@@ -899,15 +894,14 @@ create view test_times_view_total as
899894
;
900895
901896
create table test_times_report engine File(TSV, 'report/test-times.tsv')
902-
settings output_format_decimal_trailing_zeros = 1
903897
as select
904898
test,
905-
toDecimal64(real, 3),
906-
toDecimal64(total_client_time, 3),
899+
round(real, 3),
900+
round(total_client_time, 3),
907901
queries,
908-
toDecimal64(query_max, 3),
909-
toDecimal64(avg_real_per_query, 3),
910-
toDecimal64(query_min, 3),
902+
round(query_max, 3),
903+
round(avg_real_per_query, 3),
904+
round(query_min, 3),
911905
runs
912906
from (
913907
select * from test_times_view
@@ -919,21 +913,20 @@ create table test_times_report engine File(TSV, 'report/test-times.tsv')
919913
920914
-- report for all queries page, only main metric
921915
create table all_tests_report engine File(TSV, 'report/all-queries.tsv')
922-
settings output_format_decimal_trailing_zeros = 1
923916
as with
924917
-- server_time is sometimes reported as zero (if it's less than 1 ms),
925918
-- so we have to work around this to not get an error about conversion
926919
-- of NaN to decimal.
927920
(left > right ? left / right : right / left) as times_change_float,
928921
isFinite(times_change_float) as times_change_finite,
929-
toDecimal64(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
922+
round(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
930923
times_change_finite
931924
? (left > right ? '-' : '+') || toString(times_change_decimal) || 'x'
932925
: '--' as times_change_str
933926
select changed_fail, unstable_fail,
934-
toDecimal64(left, 3), toDecimal64(right, 3), times_change_str,
935-
toDecimal64(isFinite(diff) ? diff : 0, 3),
936-
toDecimal64(isFinite(stat_threshold) ? stat_threshold : 0, 3),
927+
round(left, 3), round(right, 3), times_change_str,
928+
round(isFinite(diff) ? diff : 0, 3),
929+
round(isFinite(stat_threshold) ? stat_threshold : 0, 3),
937930
test, query_index, query_display_name
938931
from queries order by test, query_index;
939932
@@ -1044,27 +1037,6 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes,
10441037
order by count() desc
10451038
;
10461039
1047-
create table metric_devation engine File(TSVWithNamesAndTypes,
1048-
'report/metric-deviation.$version.tsv')
1049-
settings output_format_decimal_trailing_zeros = 1
1050-
-- first goes the key used to split the file with grep
1051-
as select test, query_index, query_display_name,
1052-
toDecimal64(d, 3) d, q, metric
1053-
from (
1054-
select
1055-
test, query_index,
1056-
(q[3] - q[1])/q[2] d,
1057-
quantilesExact(0, 0.5, 1)(value) q, metric
1058-
from (select * from unstable_run_metrics
1059-
union all select * from unstable_run_traces
1060-
union all select * from unstable_run_metrics_2) mm
1061-
group by test, query_index, metric
1062-
having isFinite(d) and d > 0.5 and q[3] > 5
1063-
) metrics
1064-
left join query_display_names using (test, query_index)
1065-
order by test, query_index, d desc
1066-
;
1067-
10681040
create table stacks engine File(TSV, 'report/stacks.$version.tsv') as
10691041
select
10701042
-- first goes the key used to split the file with grep
@@ -1173,9 +1145,8 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as
11731145
11741146
-- Show metrics that have changed
11751147
create table changes engine File(TSV, 'metrics/changes.tsv')
1176-
settings output_format_decimal_trailing_zeros = 1
11771148
as select metric, left, right,
1178-
toDecimal64(diff, 3), toDecimal64(times_diff, 3)
1149+
round(diff, 3), round(times_diff, 3)
11791150
from (
11801151
select metric, median(left) as left, median(right) as right,
11811152
(right - left) / left diff,
@@ -1226,7 +1197,6 @@ create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
12261197
'$SHA_TO_TEST' :: LowCardinality(String) AS commit_sha,
12271198
'${CLICKHOUSE_PERFORMANCE_COMPARISON_CHECK_NAME:-Performance}' :: LowCardinality(String) AS check_name,
12281199
'$(sed -n 's/.*<!--status: \(.*\)-->/\1/p' report.html)' :: LowCardinality(String) AS check_status,
1229-
-- TODO toDateTime() can't parse output of 'date', so no time for now.
12301200
(($(date +%s) - $CHPC_CHECK_START_TIMESTAMP) * 1000) :: UInt64 AS check_duration_ms,
12311201
fromUnixTimestamp($CHPC_CHECK_START_TIMESTAMP) check_start_time,
12321202
test_name :: LowCardinality(String) AS test_name ,

‎docs/en/engines/table-engines/integrations/azureBlobStorage.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ CREATE TABLE azure_blob_storage_table (name String, value UInt32)
2121

2222
- `connection_string|storage_account_url` — connection_string includes account name & key ([Create connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&bc=%2Fazure%2Fstorage%2Fblobs%2Fbreadcrumb%2Ftoc.json#configure-a-connection-string-for-an-azure-storage-account)) or you could also provide the storage account url here and account name & account key as separate parameters (see parameters account_name & account_key)
2323
- `container_name` - Container name
24-
- `blobpath` - file path. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings.
24+
- `blobpath` - file path. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings.
2525
- `account_name` - if storage_account_url is used, then account name can be specified here
2626
- `account_key` - if storage_account_url is used, then account key can be specified here
2727
- `format` — The [format](/docs/en/interfaces/formats.md) of the file.

0 commit comments

Comments
 (0)