Skip to content

Commit 5ef2ce2

Browse files
yakov-olkhovskiyyakov-olkhovskiy
authored
Merge branch 'master' into opt-constant-node
2 parents 4260839 + ff72115 commit 5ef2ce2

File tree

546 files changed

+27799
-145552
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

546 files changed

+27799
-145552
lines changed

‎.github/PULL_REQUEST_TEMPLATE.md‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
1-
<!---
2-
A technical comment, you are free to remove or leave it as it is when PR is created
3-
The following categories are used in the next scripts, update them accordingly
4-
utils/changelog/changelog.py
5-
tests/ci/cancel_and_rerun_workflow_lambda/app.py
6-
-->
1+
<!---AI changelog entry and formatting assistance: true-->
2+
73
### Changelog category (leave one):
84
- New Feature
95
- Experimental Feature

‎.github/copilot-instructions.md‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1 @@
1-
When creating a PR, use PR body template from `/.github/PULL_REQUEST_TEMPLATE.md`
2-
31
When performing a code review, ignore `/.github/workflows/*` files.
4-
5-
When performing a code review, suggest a changelog entry for a change in a review comment,
6-
format it as plain text and limit to a maximum of 50 words.

‎.github/workflows/pull_request.yml‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,44 @@ jobs:
207207
python3 -m praktika run 'Style check' --workflow "PR" --ci |& tee ./ci/tmp/job.log
208208
fi
209209
210+
pr_formatter:
211+
runs-on: [self-hosted, style-checker-aarch64]
212+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check]
213+
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'UFIgZm9ybWF0dGVy') }}
214+
name: "PR formatter"
215+
outputs:
216+
data: ${{ steps.run.outputs.DATA }}
217+
pipeline_status: ${{ steps.run.outputs.pipeline_status }}
218+
steps:
219+
- name: Checkout code
220+
uses: actions/checkout@v4
221+
with:
222+
ref: ${{ env.CHECKOUT_REF }}
223+
224+
- name: Prepare env script
225+
run: |
226+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
227+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
228+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
229+
export PYTHONPATH=./ci:.:
230+
231+
cat > ./ci/tmp/workflow_status.json << 'EOF'
232+
${{ toJson(needs) }}
233+
EOF
234+
ENV_SETUP_SCRIPT_EOF
235+
236+
- name: Run
237+
id: run
238+
run: |
239+
echo "pipeline_status=undefined" >> $GITHUB_OUTPUT
240+
. ./ci/tmp/praktika_setup_env.sh
241+
set -o pipefail
242+
if command -v ts &> /dev/null; then
243+
python3 -m praktika run 'PR formatter' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
244+
else
245+
python3 -m praktika run 'PR formatter' --workflow "PR" --ci |& tee ./ci/tmp/job.log
246+
fi
247+
210248
docs_check:
211249
runs-on: [self-hosted, arm-medium]
212250
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check, fast_test, build_arm_tidy, build_arm_binary]
@@ -4503,7 +4541,7 @@ jobs:
45034541
45044542
finish_workflow:
45054543
runs-on: [self-hosted, style-checker-aarch64]
4506-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check, docs_check, fast_test, build_arm_tidy, build_amd_debug, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_asan, build_arm_coverage, build_arm_binary, build_amd_release, build_arm_release, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stateless_tests_amd_asan_distributed_plan_sequential, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_parallel, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_sequential, stateless_tests_amd_binary_parallelreplicas_s3_storage_parallel, stateless_tests_amd_binary_parallelreplicas_s3_storage_sequential, stateless_tests_amd_debug_asyncinsert_s3_storage_parallel, stateless_tests_amd_debug_asyncinsert_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_msan_parallel_1_2, stateless_tests_amd_msan_parallel_2_2, stateless_tests_amd_msan_sequential_1_2, stateless_tests_amd_msan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_tsan_s3_storage_parallel, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, bugfix_validation_integration_tests, bugfix_validation_functional_tests, stateless_tests_amd_asan_flaky_check, integration_tests_amd_asan_old_analyzer_1_6, integration_tests_amd_asan_old_analyzer_2_6, integration_tests_amd_asan_old_analyzer_3_6, integration_tests_amd_asan_old_analyzer_4_6, integration_tests_amd_asan_old_analyzer_5_6, integration_tests_amd_asan_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_amd_asan_flaky, docker_server_image, docker_keeper_image, install_packages_amd_debug, compatibility_check_amd_release, compatibility_check_arm_release, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_amd_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_3, performance_comparison_amd_release_master_head_2_3, performance_comparison_amd_release_master_head_3_3, performance_comparison_arm_release_master_head_1_3, performance_comparison_arm_release_master_head_2_3, performance_comparison_arm_release_master_head_3_3]
4544+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, style_check, pr_formatter, docs_check, fast_test, build_arm_tidy, build_amd_debug, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_asan, build_arm_coverage, build_arm_binary, build_amd_release, build_arm_release, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stateless_tests_amd_asan_distributed_plan_sequential, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_parallel, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_sequential, stateless_tests_amd_binary_parallelreplicas_s3_storage_parallel, stateless_tests_amd_binary_parallelreplicas_s3_storage_sequential, stateless_tests_amd_debug_asyncinsert_s3_storage_parallel, stateless_tests_amd_debug_asyncinsert_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_msan_parallel_1_2, stateless_tests_amd_msan_parallel_2_2, stateless_tests_amd_msan_sequential_1_2, stateless_tests_amd_msan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_tsan_s3_storage_parallel, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, bugfix_validation_integration_tests, bugfix_validation_functional_tests, stateless_tests_amd_asan_flaky_check, integration_tests_amd_asan_old_analyzer_1_6, integration_tests_amd_asan_old_analyzer_2_6, integration_tests_amd_asan_old_analyzer_3_6, integration_tests_amd_asan_old_analyzer_4_6, integration_tests_amd_asan_old_analyzer_5_6, integration_tests_amd_asan_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_amd_asan_flaky, docker_server_image, docker_keeper_image, install_packages_amd_debug, compatibility_check_amd_release, compatibility_check_arm_release, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_amd_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_3, performance_comparison_amd_release_master_head_2_3, performance_comparison_amd_release_master_head_3_3, performance_comparison_arm_release_master_head_1_3, performance_comparison_arm_release_master_head_2_3, performance_comparison_arm_release_master_head_3_3]
45074545
if: ${{ !cancelled() }}
45084546
name: "Finish Workflow"
45094547
outputs:

‎.gitmodules‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@
353353
url = https://github.com/ClickHouse/double-conversion.git
354354
[submodule "contrib/mongo-cxx-driver"]
355355
path = contrib/mongo-cxx-driver
356-
url = https://github.com/ClickHouse/mongo-cxx-driver.git
356+
url = https://github.com/mongodb/mongo-cxx-driver.git
357357
[submodule "contrib/mongo-c-driver"]
358358
path = contrib/mongo-c-driver
359359
url = https://github.com/ClickHouse/mongo-c-driver.git

‎base/poco/NetSSL_OpenSSL/src/Context.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ void Context::flushSessionCache()
408408
poco_assert (isForServerUse());
409409

410410
Poco::Timestamp now;
411-
SSL_CTX_flush_sessions(_pSSLContext, static_cast<long>(now.epochTime()));
411+
SSL_CTX_flush_sessions_ex(_pSSLContext, static_cast<long>(now.epochTime()));
412412
}
413413

414414

‎ci/defs/defs.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ class JobNames:
308308
DOCKER_BUILDS_ARM = "Dockers build (arm)"
309309
DOCKER_BUILDS_AMD = "Dockers build (amd)"
310310
STYLE_CHECK = "Style check"
311+
PR_BODY = "PR formatter"
311312
FAST_TEST = "Fast test"
312313
BUILD = "Build"
313314
UNITTEST = "Unit tests"
@@ -318,7 +319,7 @@ class JobNames:
318319
UPGRADE = "Upgrade check"
319320
PERFORMANCE = "Performance Comparison"
320321
COMPATIBILITY = "Compatibility check"
321-
Docs = "Docs check"
322+
DOCS = "Docs check"
322323
CLICKBENCH = "ClickBench"
323324
DOCKER_SERVER = "Docker server image"
324325
DOCKER_KEEPER = "Docker keeper image"

‎ci/defs/job_configs.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,13 @@ class JobConfigs:
115115
run_in_docker="clickhouse/style-test",
116116
enable_commit_status=True,
117117
)
118+
pr_body = Job.Config(
119+
name=JobNames.PR_BODY,
120+
runs_on=RunnerLabels.STYLE_CHECK_ARM,
121+
command="python3 ./ci/jobs/pr_formatter_job.py",
122+
allow_merge_on_failure=True,
123+
enable_gh_auth=True,
124+
)
118125
fast_test = Job.Config(
119126
name=JobNames.FAST_TEST,
120127
runs_on=RunnerLabels.AMD_LARGE,
@@ -925,7 +932,7 @@ class JobConfigs:
925932
),
926933
)
927934
docs_job = Job.Config(
928-
name=JobNames.Docs,
935+
name=JobNames.DOCS,
929936
runs_on=RunnerLabels.FUNC_TESTER_ARM,
930937
command="python3 ./ci/jobs/docs_job.py",
931938
digest_config=Job.CacheDigestConfig(

‎ci/jobs/pr_formatter_job.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import os
2+
import shlex
3+
4+
from ci.praktika import Secret
5+
from ci.praktika.gh import GH
6+
from ci.praktika.info import Info
7+
from ci.praktika.result import Result
8+
9+
if __name__ == "__main__":
10+
info = Info()
11+
assert info.pr_number, "This job must run for a Pull Request"
12+
PR_BODY_TEMPLATE_PATH = ".github/PULL_REQUEST_TEMPLATE.md"
13+
output_file = "./ci/tmp/pr_body_generated.md"
14+
15+
prompt = f"""
16+
Generate a formatted PR body for PR #{info.pr_number} following ClickHouse conventions.
17+
18+
## Required Sections
19+
20+
### 1. Changelog Category
21+
Header: "### Changelog category (leave one):"
22+
- If a category is already selected in the PR body: keep only that ONE category
23+
- If no category is selected: include ALL categories below so the user can choose later
24+
- Categories must be copied EXACTLY as written (character-for-character, including parentheses):
25+
- New Feature
26+
- Experimental Feature
27+
- Improvement
28+
- Performance Improvement
29+
- Backward Incompatible Change
30+
- Build/Testing/Packaging Improvement
31+
- Documentation (changelog entry is not required)
32+
- Critical Bug Fix (crash, data loss, RBAC) or LOGICAL_ERROR
33+
- Bug Fix (user-visible misbehavior in an official stable release)
34+
- CI Fix or Improvement (changelog entry is not required)
35+
- Not for changelog (changelog entry is not required)
36+
37+
### 2. Changelog Entry
38+
Header: "### Changelog entry (a [user-readable short description](https://github.com/ClickHouse/ClickHouse/blob/master/docs/changelog_entry_guidelines.md) of the changes that goes into CHANGELOG.md):"
39+
- Include this section UNLESS the category says "changelog entry is not required"
40+
- Reuse existing entry with grammar fixes, or generate from PR title if missing
41+
- Keep it plain text, ≤50 words, no quotes or markdown formatting
42+
- If PR mentions "Fixes/Resolves/Closes #NUMBER", append: "Resolves #NUMBER"
43+
44+
## Optional Sections
45+
46+
### 3. Documentation Entry
47+
Header: "### Documentation entry for user-facing changes"
48+
- Include only if present in original PR body
49+
50+
### 4. Details
51+
Header: "### Details"
52+
- Add any additional context from the original PR body
53+
- Include issue link if referenced. for example: "Resolves #NUMBER"
54+
55+
## Formatting
56+
- Separate all sections with blank lines
57+
- Remove all markdown comments
58+
59+
## Output
60+
- Write the final PR body to ./ci/tmp/pr_body_generated.md (do not update the actual PR).
61+
- Do not print the PR body to stdout; only brief status logs if needed.
62+
- Do not print non-ANSI characters in the logs to avoid encoding issues.
63+
"""
64+
65+
res = True
66+
results = []
67+
68+
os.environ["GH_TOKEN"] = Secret.Config(
69+
name="/github-tokens/robot-2-copilot", type=Secret.Type.AWS_SSM_PARAMETER
70+
).get_value()
71+
if res:
72+
results.append(
73+
Result.from_commands_run(
74+
name="prompt",
75+
command=f"copilot -p {shlex.quote(prompt)} --allow-all-tools",
76+
with_info=True,
77+
)
78+
)
79+
res = results[-1].is_ok()
80+
# unset from env to post from default gh app
81+
os.environ.pop("GH_TOKEN", None)
82+
83+
if res:
84+
results.append(
85+
Result.from_commands_run(
86+
name="check output",
87+
command=[
88+
f"cat {output_file}",
89+
f"test -f {output_file} && test $(wc -l < {output_file}) -gt 1 && test $(wc -c < {output_file}) -gt 50",
90+
f"sed -i.bak '1s/^/<!---AI changelog entry and formatting assistance: false-->\\n/' {output_file} && rm -f {output_file}.bak",
91+
],
92+
)
93+
)
94+
res = results[-1].is_ok()
95+
if res:
96+
results.append(
97+
Result.from_commands_run(
98+
name="update PR body",
99+
command=[lambda: GH.update_pr_body(body_file=output_file)],
100+
)
101+
)
102+
res = results[-1].is_ok()
103+
104+
Result.create_from(results=results).complete_job()

‎ci/jobs/scripts/workflow_hooks/can_be_merged.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import sys
22

3-
from ci.jobs.scripts.workflow_hooks.pr_description import Labels
3+
from ci.jobs.scripts.workflow_hooks.pr_labels_and_category import Labels
44
from ci.praktika.info import Info
55

66

‎ci/jobs/scripts/workflow_hooks/feature_docs.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
from praktika.info import Info
55

6-
from ci.jobs.scripts.workflow_hooks.pr_description import Labels
6+
from ci.jobs.scripts.workflow_hooks.pr_labels_and_category import Labels
77

88
autogenerated_doc_files = [
99
"src/Storages/MergeTree/MergeTreeSettings.cpp",

0 commit comments

Comments
 (0)