Skip to content

Commit 37f3ea6

Browse files
alesapinalesapin
committed
Merge remote-tracking branch 'origin/master' into disable_catalogs_in_system_tables
2 parents 3232b8c + f6cb150 commit 37f3ea6

File tree

178 files changed

+5352
-2373
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

178 files changed

+5352
-2373
lines changed

‎ci/docker/integration/arrowflight/flight_server.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
#!/usr/bin/env python3
22

3-
import pyarrow as pa
4-
import pyarrow.flight as fl
53
import argparse
64
import base64
75
import json
86

7+
import pyarrow as pa
8+
import pyarrow.flight as fl
9+
910

1011
class FlightServer(fl.FlightServerBase):
1112
def __init__(self, location, auth_handler, middleware):

‎ci/jobs/clickbench.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@ def main():
1111
results = []
1212
stop_watch = Utils.Stopwatch()
1313
ch = ClickHouseProc()
14+
info = Info()
1415

1516
if res:
1617
print("Install ClickHouse")
1718

1819
def install():
1920
res = ch.install_clickbench_config()
20-
if Info().is_local_run:
21+
if info.is_local_run:
2122
return res
2223
return res and ch.create_log_export_config()
2324

@@ -31,7 +32,7 @@ def install():
3132

3233
def start():
3334
res = ch.start_light()
34-
if Info().is_local_run:
35+
if info.is_local_run:
3536
return res
3637
return res and ch.start_log_exports(check_start_time=stop_watch.start_time)
3738

@@ -96,7 +97,9 @@ def start():
9697
)
9798

9899
Result.create_from(
99-
results=results, stopwatch=stop_watch, files=ch.prepare_logs(all=False)
100+
results=results,
101+
stopwatch=stop_watch,
102+
files=ch.prepare_logs(all=False, info=info),
100103
).complete_job()
101104

102105

‎ci/jobs/functional_tests.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ def start():
418418
print("Collect logs")
419419

420420
def collect_logs():
421-
CH.prepare_logs(all=test_result and not test_result.is_ok())
421+
CH.prepare_logs(all=test_result and not test_result.is_ok(), info=info)
422422

423423
results.append(
424424
Result.from_commands_run(

‎ci/jobs/fuzzers_job.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from praktika.utils import Shell, Utils
33

44
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
5+
from ci.praktika.info import Info
56

67
temp_dir = f"{Utils.cwd()}/ci/tmp/"
78

@@ -11,6 +12,7 @@ def main():
1112
results = []
1213
stop_watch = Utils.Stopwatch()
1314
ch = ClickHouseProc()
15+
info = Info()
1416

1517
if res:
1618
print("Install ClickHouse")
@@ -53,7 +55,7 @@ def start():
5355
)
5456

5557
Result.create_from(
56-
results=results, stopwatch=stop_watch, files=[ch.prepare_logs()]
58+
results=results, stopwatch=stop_watch, files=[ch.prepare_logs(info=info)]
5759
).complete_job()
5860

5961

‎ci/jobs/integration_test_job.py‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ def main():
113113
use_distributed_plan = True
114114
elif to == "flaky":
115115
is_flaky_check = True
116-
repeat_option = f"--count {FLAKY_CHECK_TEST_REPEAT_COUNT} --random-order"
116+
args.count = FLAKY_CHECK_TEST_REPEAT_COUNT
117117
elif to == "parallel":
118118
is_parallel = True
119119
elif to == "sequential":
@@ -124,7 +124,7 @@ def main():
124124
assert False, f"Unknown job option [{to}]"
125125

126126
if args.count:
127-
repeat_option = f"--count {args.count}"
127+
repeat_option = f"--count {args.count} --random-order"
128128

129129
changed_test_modules = []
130130
if is_bugfix_validation or is_flaky_check:
@@ -156,14 +156,21 @@ def main():
156156
if args.path:
157157
clickhouse_path = args.path
158158
else:
159-
if Path(clickhouse_path).is_file():
160-
pass
161-
elif Path(f"{Utils.cwd()}/build/programs/clickhouse").is_file():
162-
clickhouse_path = f"{Utils.cwd()}/build/programs/clickhouse"
163-
elif Path(f"{Utils.cwd()}/clickhouse").is_file():
164-
clickhouse_path = f"{Utils.cwd()}/clickhouse"
159+
paths_to_check = [
160+
clickhouse_path, # it's set for CI runs, but we need to check it
161+
f"{Utils.cwd()}/build/programs/clickhouse",
162+
f"{Utils.cwd()}/clickhouse",
163+
]
164+
for path in paths_to_check:
165+
if Path(path).is_file():
166+
clickhouse_path = path
167+
break
165168
else:
166-
raise FileNotFoundError(f"Clickhouse binary not found")
169+
raise FileNotFoundError(
170+
"Clickhouse binary not found in any of the paths: "
171+
+ ", ".join(paths_to_check)
172+
+ ". You can also specify path to binary via --path argument"
173+
)
167174
if args.path_1:
168175
clickhouse_server_config_dir = args.path_1
169176
assert Path(

‎ci/jobs/scripts/clickhouse_proc.py‎

Lines changed: 59 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import traceback
77
from collections import defaultdict
88
from pathlib import Path
9+
import uuid
910

1011
from ci.praktika import Secret
1112
from ci.praktika.info import Info
@@ -248,6 +249,9 @@ def start_light(self):
248249
print(f"ClickHouse server ready")
249250
else:
250251
print(f"ClickHouse server NOT ready")
252+
253+
self._flush_system_logs()
254+
self.save_system_metadata_files_from_remote_database_disk()
251255
return res
252256

253257
def install_clickbench_config(self):
@@ -482,6 +486,9 @@ def start(self, replica_num=0):
482486
if self.is_db_replicated and replica_num == 0:
483487
res = self.start(replica_num=1) and self.start(replica_num=2)
484488

489+
self._flush_system_logs()
490+
self.save_system_metadata_files_from_remote_database_disk()
491+
485492
return res
486493

487494
def create_minio_log_tables(self):
@@ -692,27 +699,35 @@ def terminate(self):
692699

693700
return self
694701

695-
def prepare_logs(self, all=False):
696-
res = self._get_logs_archives_server()
697-
res += self._get_jemalloc_profiles()
698-
if Path(self.GDB_LOG).exists():
699-
res.append(self.GDB_LOG)
700-
if all:
701-
res += self.debug_artifacts
702-
res += self.dump_system_tables()
703-
res += self._collect_core_dumps()
704-
res += self._get_logs_archive_coordination()
705-
if Path(self.MINIO_LOG).exists():
706-
res.append(self.MINIO_LOG)
707-
if Path(self.AZURITE_LOG).exists():
708-
res.append(self.AZURITE_LOG)
709-
if Path(self.DMESG_LOG).exists():
710-
res.append(self.DMESG_LOG)
711-
if Path(self.CH_LOCAL_ERR_LOG).exists():
712-
res.append(self.CH_LOCAL_ERR_LOG)
713-
if Path(self.CH_LOCAL_LOG).exists():
714-
res.append(self.CH_LOCAL_LOG)
715-
self.logs = res
702+
def prepare_logs(self, info, all=False):
703+
res = []
704+
try:
705+
res = self._get_logs_archives_server()
706+
res += self._get_jemalloc_profiles()
707+
if Path(self.GDB_LOG).exists():
708+
res.append(self.GDB_LOG)
709+
if all:
710+
res += self.debug_artifacts
711+
res += self.dump_system_tables()
712+
res += self._collect_core_dumps()
713+
res += self._get_logs_archive_coordination()
714+
if Path(self.MINIO_LOG).exists():
715+
res.append(self.MINIO_LOG)
716+
if Path(self.AZURITE_LOG).exists():
717+
res.append(self.AZURITE_LOG)
718+
if Path(self.DMESG_LOG).exists():
719+
res.append(self.DMESG_LOG)
720+
if Path(self.CH_LOCAL_ERR_LOG).exists():
721+
res.append(self.CH_LOCAL_ERR_LOG)
722+
if Path(self.CH_LOCAL_LOG).exists():
723+
res.append(self.CH_LOCAL_LOG)
724+
self.logs = res
725+
except Exception as e:
726+
print(f"WARNING: Failed to collect logs: {e}")
727+
traceback.print_exc()
728+
info.add_workflow_report_message(
729+
f"Failed to collect all logs in job [{info.job_name}], ex [{e}], see job.log"
730+
)
716731
return res
717732

718733
def _collect_core_dumps(self):
@@ -1052,23 +1067,41 @@ def dump_system_tables(self):
10521067
res = False
10531068
return [f for f in glob.glob(f"{temp_dir}/system_tables/*.tsv")]
10541069

1070+
@staticmethod
1071+
def is_valid_uuid(val):
1072+
try:
1073+
uuid_obj = uuid.UUID(val)
1074+
return str(uuid_obj) == val.lower()
1075+
except ValueError:
1076+
return False
1077+
10551078
def save_system_metadata_files_from_remote_database_disk(self):
10561079
if not os.path.exists(
10571080
"/etc/clickhouse-server/config.d/remote_database_disk.xml"
10581081
):
10591082
return
10601083

10611084
# Store system database and table metadata files
1062-
self.system_db_uuid = Shell.get_output(
1085+
system_db_uuid = Shell.get_output(
10631086
"clickhouse disks -C /etc/clickhouse-server/config.xml --disk disk_db_remote -q 'read metadata/system.sql' | grep -F UUID | awk -F\"'\" '{print $2}'",
10641087
verbose=True,
10651088
)
1089+
if not self.is_valid_uuid(system_db_uuid):
1090+
print(f"invalid system_db_uuid: '{system_db_uuid}'")
1091+
return
1092+
1093+
if self.system_db_uuid != None and self.system_db_uuid != system_db_uuid:
1094+
print(
1095+
f"system_db_uuid changed: '{self.system_db_uuid}' -> '{system_db_uuid}'"
1096+
)
1097+
1098+
self.system_db_uuid = system_db_uuid
10661099
self.system_db_sql = Shell.get_output(
10671100
"clickhouse disks -C /etc/clickhouse-server/config.xml --disk disk_db_remote -q 'read metadata/system.sql'",
10681101
verbose=True,
10691102
)
1070-
print(f"system_db_uuid = {self.system_db_uuid}")
1071-
print(f"system_db_sql = {self.system_db_sql}")
1103+
print(f"system_db_uuid = '{self.system_db_uuid}'")
1104+
print(f"system_db_sql = '{self.system_db_sql}'")
10721105

10731106
system_table_sql_files = (
10741107
Shell.get_output(
@@ -1080,6 +1113,7 @@ def save_system_metadata_files_from_remote_database_disk(self):
10801113
)
10811114
self.system_table_sql_map = {}
10821115
for system_table_sql_file in system_table_sql_files:
1116+
print(f"system_table_sql_file = '{system_table_sql_file}'")
10831117
sql_content = Shell.get_output(
10841118
f"clickhouse disks -C /etc/clickhouse-server/config.xml --disk disk_db_remote -q 'read store/{self.system_db_uuid[:3]}/{self.system_db_uuid}/{system_table_sql_file}'",
10851119
verbose=True,
@@ -1097,7 +1131,7 @@ def restore_system_metadata_files_from_remote_database_disk(self):
10971131
# Restore system database and table metadata files for `clickhouse local`
10981132
with open(f"{self.run_path0}/metadata/system.sql", "w") as file:
10991133
file.write(self.system_db_sql)
1100-
res = Shell.check(
1134+
Shell.check(
11011135
f"mkdir -p {self.run_path0}/store/{self.system_db_uuid[:3]}/{self.system_db_uuid}",
11021136
verbose=True,
11031137
)

‎ci/jobs/vector_search_stress_tests.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,18 @@
22
# Documentation : https://clickhouse.com/docs/engines/table-engines/mergetree-family/annindexes
33

44
import os
5+
import random
56
import sys
7+
import threading
8+
import time
69
import traceback
10+
711
import clickhouse_connect
8-
import random
9-
import time
10-
import threading
1112
import numpy as np
12-
from ci.praktika.result import Result
13+
1314
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
1415
from ci.praktika.info import Info
16+
from ci.praktika.result import Result
1517
from ci.praktika.utils import Shell, Utils
1618

1719
temp_dir = f"{Utils.cwd()}/ci/tmp/"
@@ -178,6 +180,7 @@
178180
CONCURRENCY_TEST: True,
179181
}
180182

183+
181184
def get_new_connection():
182185
chclient = clickhouse_connect.get_client(send_receive_timeout=1800)
183186
return chclient
@@ -598,7 +601,7 @@ def start():
598601
"Test using the hackernews dataset",
599602
dataset_hackernews_openai,
600603
test_params_hackernews_10m,
601-
)
604+
),
602605
]
603606

604607

‎ci/praktika/cidb.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def __init__(self, url, user, passwd):
5555
}
5656

5757
def get_link_to_test_case_statistics(
58-
self, test_name: str, job_name: Optional[str] = None, user: Optional[str] = None
58+
self, test_name: str, job_name: Optional[str] = None, url="", user=""
5959
) -> str:
6060
"""
6161
Build a link to query CI DB statistics for a specific test case.
@@ -87,7 +87,7 @@ def get_link_to_test_case_statistics(
8787
"""
8888

8989
# Compose base URL, optionally attaching user parameter
90-
base = self.url or ""
90+
base = url or self.url or ""
9191
if user:
9292
sep = "&" if "?" in base else "?"
9393
base = f"{base}/play{sep}user={urllib.parse.quote(user, safe='')}&run=1"

‎ci/praktika/gh.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,7 @@ def extract_hlabels_info(res: Result) -> str:
458458
if len(summary.failed_results) > MAX_JOBS_PER_SUMMARY:
459459
remaining = len(summary.failed_results) - MAX_JOBS_PER_SUMMARY
460460
summary.failed_results = summary.failed_results[:MAX_JOBS_PER_SUMMARY]
461-
print(f"NOTE: {remaining} more jobs not shown in PR comment", status="")
461+
print(f"NOTE: {remaining} more jobs not shown in PR comment")
462462
return summary
463463

464464
def to_markdown(self):

‎ci/praktika/hook_html.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ def add_dependees(job_name):
289289

290290
for dependee in dependees:
291291
print(
292-
f"NOTE: Set job [{dependee}] status to [{Result.Status.SKIPPED}] due to current failure"
292+
f"NOTE: Set job [{dependee}] status to [{Result.Status.DROPPED}] due to current failure"
293293
)
294294
new_sub_results.append(
295295
Result(

0 commit comments

Comments
 (0)