Skip to content

Commit b9d11e6

Browse files
yariks5syariks5s
committed
Merge remote-tracking branch 'origin/master' into yarik/fix-todatetime-null-bug
2 parents d5ddf49 + f0fa9dc commit b9d11e6

File tree

301 files changed

+6100
-2601
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

301 files changed

+6100
-2601
lines changed

‎.github/workflows/nightly_jepsen.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ jobs:
179179
fi
180180
181181
clickhouse_keeper_jepsen:
182-
runs-on: [self-hosted, style-checker-aarch64]
182+
runs-on: [self-hosted, style-checker]
183183
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary]
184184
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'Q2xpY2tIb3VzZSBLZWVwZXIgSmVwc2Vu') }}
185185
name: "ClickHouse Keeper Jepsen"

‎ci/defs/job_configs.py‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,12 @@
7070
common_stress_job_config = Job.Config(
7171
name=JobNames.STRESS,
7272
runs_on=[], # from parametrize()
73-
command="cd ./tests/ci && python3 ./stress_check.py",
73+
command="python3 ./ci/jobs/stress_job.py",
7474
digest_config=Job.CacheDigestConfig(
7575
include_paths=[
7676
"./tests/queries/0_stateless/",
77-
"./tests/ci/stress.py",
78-
"./tests/ci/stress_check.py",
77+
"./ci/jobs/stress_job.py",
78+
"./ci/jobs/scripts/stress/stress.py",
7979
"./tests/clickhouse-test",
8080
"./tests/config",
8181
"./tests/*.txt",
@@ -680,11 +680,12 @@ class JobConfigs:
680680
upgrade_test_jobs = Job.Config(
681681
name=JobNames.UPGRADE,
682682
runs_on=["from param"],
683-
command="cd ./tests/ci && python3 ./upgrade_check.py",
683+
command="python3 ./ci/jobs/upgrade_job.py",
684684
digest_config=Job.CacheDigestConfig(
685685
include_paths=[
686-
"./tests/ci/upgrade_check.py",
687-
"./tests/ci/stress_check.py",
686+
"./ci/jobs/upgrade_job.py",
687+
"./ci/jobs/stress_job.py",
688+
"./ci/jobs/scripts/stress/stress.py",
688689
"./tests/docker_scripts/",
689690
"./ci/docker/stress-test",
690691
]
@@ -1047,13 +1048,13 @@ class JobConfigs:
10471048
jepsen_keeper = Job.Config(
10481049
name=JobNames.JEPSEN_KEEPER,
10491050
runs_on=RunnerLabels.STYLE_CHECK_AMD,
1050-
command="cd ./tests/ci && python3 ci.py --run-from-praktika",
1051+
command="python3 ./ci/jobs/jepsen_check.py keeper",
10511052
requires=["Build (amd_binary)"],
10521053
)
10531054
jepsen_server = Job.Config(
10541055
name=JobNames.JEPSEN_KEEPER,
10551056
runs_on=RunnerLabels.STYLE_CHECK_AMD,
1056-
command="cd ./tests/ci && python3 ci.py --run-from-praktika",
1057+
command="python3 ./ci/jobs/jepsen_check.py server",
10571058
requires=["Build (amd_binary)"],
10581059
)
10591060
libfuzzer_job = Job.Config(

‎ci/jobs/buzzhouse_job.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,8 @@ def main():
175175
"truncate_output": True,
176176
# Don't always run transactions, makes many statements fail
177177
"allow_transactions": random.randint(1, 5) == 1,
178+
# Run query oracles sometimes
179+
"allow_query_oracles": random.randint(1, 4) == 1,
178180
"remote_servers": ["localhost:9000"],
179181
"remote_secure_servers": ["localhost:9440"],
180182
"http_servers": ["localhost:8123"],

‎ci/jobs/functional_tests.py‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ def run_tests(
118118
"parallel": "--no-sequential",
119119
"sequential": "--no-parallel",
120120
"flaky check": "--flaky-check",
121+
"targeted": "--flaky-check", # to disable tests not compatible with the thread fuzzer
121122
}
122123

123124

@@ -147,12 +148,7 @@ def main():
147148
elif to in OPTIONS_TO_INSTALL_ARGUMENTS:
148149
print(f"NOTE: Enabled config option [{OPTIONS_TO_INSTALL_ARGUMENTS[to]}]")
149150
config_installs_args += f" {OPTIONS_TO_INSTALL_ARGUMENTS[to]}"
150-
elif (
151-
to.startswith("amd_")
152-
or to.startswith("arm_")
153-
or "flaky" in to
154-
or "targeted" in to
155-
):
151+
elif to.startswith("amd_") or to.startswith("arm_"):
156152
pass
157153
elif to in OPTIONS_TO_TEST_RUNNER_ARGUMENTS:
158154
print(
@@ -218,7 +214,7 @@ def main():
218214
rerun_count = 50
219215
elif is_targeted_check:
220216
print(f"Rerun count set to 5 for targeted check")
221-
rerun_count = 10
217+
rerun_count = 5
222218

223219
if not info.is_local_run:
224220
# TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled

tests/ci/jepsen_check.py renamed to ci/jobs/jepsen_check.py

Lines changed: 58 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,62 @@
1-
#!/usr/bin/env python3
2-
31
import argparse
2+
import json
43
import logging
54
import os
65
import sys
76
import time
87
from pathlib import Path
98
from typing import Any, List
109

11-
import boto3 # type: ignore
10+
import boto3
11+
from tests.ci.ssh import SSHKey
1212

13-
from build_download_helper import read_build_urls
14-
from ci_config import CI
15-
from compress_files import compress_fast
16-
from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
17-
from get_robot_token import get_parameter_from_ssm
18-
from pr_info import PRInfo
19-
from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults
20-
from ssh import SSHKey
21-
from stopwatch import Stopwatch
22-
from tee_popen import TeePopen
13+
from ci.defs.defs import JobNames
14+
from ci.praktika.info import Info
15+
from ci.praktika.result import Result
16+
from ci.praktika.secret import Secret
17+
from ci.praktika.utils import Shell, Utils
2318

2419
JEPSEN_GROUP_NAME = "jepsen_group"
2520

2621
KEEPER_DESIRED_INSTANCE_COUNT = 3
2722
SERVER_DESIRED_INSTANCE_COUNT = 4
2823

2924
KEEPER_IMAGE_NAME = "clickhouse/keeper-jepsen-test"
30-
KEEPER_CHECK_NAME = CI.JobNames.JEPSEN_KEEPER
25+
KEEPER_CHECK_NAME = JobNames.JEPSEN_KEEPER
3126

3227
SERVER_IMAGE_NAME = "clickhouse/server-jepsen-test"
33-
SERVER_CHECK_NAME = CI.JobNames.JEPSEN_SERVER
28+
SERVER_CHECK_NAME = JobNames.JEPSEN_SERVER
3429

3530
SUCCESSFUL_TESTS_ANCHOR = "# Successful tests"
3631
INTERMINATE_TESTS_ANCHOR = "# Indeterminate tests"
3732
CRASHED_TESTS_ANCHOR = "# Crashed tests"
3833
FAILED_TESTS_ANCHOR = "# Failed tests"
3934

4035

41-
def _parse_jepsen_output(path: Path) -> TestResults:
42-
test_results = [] # type: TestResults
36+
def read_build_urls(path, build_name: str):
37+
artifact_report = path / f"artifact_report_build_{build_name}.json"
38+
if artifact_report.is_file():
39+
with open(artifact_report, "r", encoding="utf-8") as f:
40+
return json.load(f)["build_urls"]
41+
return []
42+
43+
44+
def _parse_jepsen_output(path: Path):
45+
test_results = []
4346
current_type = ""
4447
with open(path, "r", encoding="utf-8") as f:
4548
for line in f:
4649
if SUCCESSFUL_TESTS_ANCHOR in line:
47-
current_type = "OK"
50+
current_type = Result.StatusExtended.OK
4851
elif INTERMINATE_TESTS_ANCHOR in line or CRASHED_TESTS_ANCHOR in line:
49-
current_type = "ERROR"
52+
current_type = Result.StatusExtended.ERROR
5053
elif FAILED_TESTS_ANCHOR in line:
51-
current_type = "FAIL"
54+
current_type = Result.StatusExtended.FAIL
5255

5356
if (
5457
line.startswith("store/clickhouse") or line.startswith("clickhouse")
5558
) and current_type:
56-
test_results.append(TestResult(line.strip(), current_type))
59+
test_results.append(Result(line.strip(), current_type))
5760

5861
return test_results
5962

@@ -138,7 +141,7 @@ def get_run_command(
138141
):
139142
return (
140143
f"docker run --network=host -v '{ssh_sock_dir}:{ssh_sock_dir}' -e SSH_AUTH_SOCK={ssh_auth_sock} "
141-
f"-e PR_TO_TEST={pr_info.number} -e SHA_TO_TEST={pr_info.sha} -v '{nodes_path}:/nodes.txt' -v {result_path}:/test_output "
144+
f"-e PR_TO_TEST={pr_info.pr_number} -e SHA_TO_TEST={pr_info.sha} -v '{nodes_path}:/nodes.txt' -v {result_path}:/test_output "
142145
f"-e 'CLICKHOUSE_PACKAGE={build_url}' -v '{repo_path}:/ch' -e 'CLICKHOUSE_REPO_PATH=/ch' -e NODES_USERNAME=ubuntu {extra_args} {docker_image}"
143146
)
144147

@@ -158,27 +161,24 @@ def main():
158161
logging.warning("Invalid argument '%s'", args.program)
159162
sys.exit(0)
160163

161-
stopwatch = Stopwatch()
162-
temp_path = Path(TEMP_PATH)
164+
stopwatch = Utils.Stopwatch()
165+
temp_path = Path(Utils.cwd()) / "ci/tmp"
163166
temp_path.mkdir(parents=True, exist_ok=True)
164167

165-
pr_info = PRInfo()
168+
info = Info()
166169

167170
logging.info(
168171
"Start at PR number %s, commit sha %s labels %s",
169-
pr_info.number,
170-
pr_info.sha,
171-
pr_info.labels,
172+
info.pr_number,
173+
info.sha,
174+
info.pr_labels,
172175
)
173176

174-
if pr_info.number != 0 and "jepsen-test" not in pr_info.labels:
177+
if info.pr_number != 0 and "jepsen-test" not in info.pr_labels:
175178
logging.info("Not jepsen test label in labels list, skipping")
176179
sys.exit(0)
177180

178-
check_name = KEEPER_CHECK_NAME if args.program == "keeper" else SERVER_CHECK_NAME
179-
180-
if not os.path.exists(TEMP_PATH):
181-
os.makedirs(TEMP_PATH)
181+
temp_path.mkdir(parents=True, exist_ok=True)
182182

183183
result_path = temp_path / "result_path"
184184
result_path.mkdir(parents=True, exist_ok=True)
@@ -195,9 +195,8 @@ def main():
195195
# always use latest
196196
docker_image = KEEPER_IMAGE_NAME if args.program == "keeper" else SERVER_IMAGE_NAME
197197

198-
# binary_release assumed to be always ready on the master as it's part of the merge queue workflow
199-
build_name = CI.get_required_build_name(check_name)
200-
urls = read_build_urls(build_name, REPORT_PATH)
198+
# build amd_binary assumed to be always ready on the master as it's part of the merge queue workflow
199+
urls = read_build_urls(temp_path, "amd_binary")
201200
build_url = None
202201
for url in urls:
203202
if url.endswith("clickhouse"):
@@ -208,63 +207,57 @@ def main():
208207
if args.program == "server":
209208
extra_args = f"-e KEEPER_NODE={instances[-1]}"
210209

211-
with SSHKey(key_value=get_parameter_from_ssm("jepsen_ssh_key") + "\n"):
210+
ssh_key_secret = Secret.Config(
211+
name="jepsen_ssh_key", type=Secret.Type.AWS_SSM_PARAMETER
212+
)
213+
with SSHKey(key_value=ssh_key_secret.get_value() + "\n"):
212214
ssh_auth_sock = os.environ["SSH_AUTH_SOCK"]
213215
auth_sock_dir = os.path.dirname(ssh_auth_sock)
214216
cmd = get_run_command(
215217
ssh_auth_sock,
216218
auth_sock_dir,
217-
pr_info,
219+
info,
218220
nodes_path,
219-
REPO_COPY,
221+
Utils.cwd(),
220222
build_url,
221223
result_path,
222224
extra_args,
223225
docker_image,
224226
)
225227
logging.info("Going to run jepsen: %s", cmd)
226228

227-
run_log_path = temp_path / "run.log"
229+
if Shell.run(cmd) != 0:
230+
logging.error("Jepsen run command failed")
228231

229-
with TeePopen(cmd, run_log_path) as process:
230-
retcode = process.wait()
231-
if retcode == 0:
232-
logging.info("Run successfully")
233-
else:
234-
logging.info("Run failed")
232+
clear_autoscaling_group()
235233

236-
status = SUCCESS
234+
status = Result.Status.SUCCESS
237235
description = "No invalid analysis found ヽ(‘ー`)ノ"
238236
jepsen_log_path = result_path / "jepsen_run_all_tests.log"
239237
additional_data = []
240238
try:
241239
test_result = _parse_jepsen_output(jepsen_log_path)
242240
if len(test_result) == 0:
243-
status = FAILURE
241+
status = Result.Status.FAILED
244242
description = "No test results found"
245243
elif any(r.status == "FAIL" for r in test_result):
246-
status = FAILURE
244+
status = Result.Status.FAILED
247245
description = "Found invalid analysis (ﾉಥ益ಥ）ﾉ ┻━┻"
248246

249-
compress_fast(result_path / "store", result_path / "jepsen_store.tar.zst")
250-
additional_data.append(result_path / "jepsen_store.tar.zst")
247+
additional_data.append(Utils.compress_zst(result_path / "store"))
251248
except Exception as ex:
252249
print("Exception", ex)
253-
status = FAILURE
250+
status = Result.Status.FAILED
254251
description = "No Jepsen output log"
255-
test_result = [TestResult("No Jepsen output log", "FAIL")]
256-
257-
JobReport(
258-
description=description,
259-
test_results=test_result,
260-
status=status,
261-
start_time=stopwatch.start_time_str,
262-
duration=stopwatch.duration_seconds,
263-
additional_files=[run_log_path] + additional_data,
264-
check_name=check_name,
265-
).dump()
266-
267-
clear_autoscaling_group()
252+
test_result = [Result("No Jepsen output log", "FAIL")]
253+
254+
Result.create_from(
255+
results=test_result,
256+
status=status if not test_result else "",
257+
stopwatch=stopwatch,
258+
files=additional_data,
259+
info=description,
260+
).complete_job()
268261

269262

270263
if __name__ == "__main__":

‎ci/jobs/scripts/check_style/check_cpp.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
LC_ALL="en_US.UTF-8"
1616
ROOT_PATH=$(git rev-parse --show-toplevel)
17-
EXCLUDE='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|programs/keeper-bench/example.yaml|base/base/openpty.h|src/Storages/ObjectStorage/DataLakes/Iceberg/AvroSchema.h'
17+
EXCLUDE='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*\.cpp$|programs/keeper-bench/example\.yaml|base/base/openpty\.h|src/Storages/ObjectStorage/DataLakes/Iceberg/AvroSchema\.h'
1818
EXCLUDE_DOCS='Settings\.cpp|FormatFactorySettings\.h'
1919

2020
# From [1]:

0 commit comments

Comments
 (0)