ClickHouse · Felixoid · May 3, 2024 · Apr 12, 2024 · Apr 24, 2024 · May 3, 2024
diff --git a/tests/ci/ci.py b/tests/ci/ci.py
@@ -17,7 +17,7 @@
 import docker_images_helper
 import upload_result_helper
 from build_check import get_release_or_pr
-from ci_config import CI_CONFIG, Build, CILabels, CIStages, JobNames
+from ci_config import CI_CONFIG, Build, CILabels, CIStages, JobNames, StatusNames
 from ci_utils import GHActions, is_hex, normalize_string
 from clickhouse_helper import (
     CiLogsCredentials,
@@ -32,15 +32,19 @@
     RerunHelper,
     format_description,
     get_commit,
+    get_commit_filtered_statuses,
     post_commit_status,
     set_status_comment,
+    trigger_mergeable_check,
     update_mergeable_check,
 )
 from digest_helper import DockerDigester, JobDigester
 from env_helper import (
     CI,
     GITHUB_JOB_API_URL,
+    GITHUB_REPOSITORY,
     GITHUB_RUN_URL,
+    GITHUB_UPSTREAM_REPOSITORY,
     REPO_COPY,
     REPORT_PATH,
     S3_BUILDS_BUCKET,
@@ -51,8 +55,9 @@
 from git_helper import Runner as GitRunner
 from github_helper import GitHub
 from pr_info import PRInfo
-from report import ERROR, SUCCESS, BuildResult, JobReport
+from report import ERROR, SUCCESS, BuildResult, JobReport, get_status
 from s3_helper import S3Helper
+from synchronizer_utils import SYNC_BRANCH_PREFIX
 from version_helper import get_version_from_repo
 
 # pylint: disable=too-many-lines
@@ -2106,6 +2111,7 @@ def main() -> int:
                 check_url = log_url
             else:
                 # test job
+                gh = GitHub(get_best_robot_token(), per_page=100)
                 additional_urls = []
                 s3_path_prefix = "/".join(
                     (
@@ -2133,9 +2139,7 @@ def main() -> int:
                         job_report.check_name or _get_ext_check_name(args.job_name),
                         additional_urls=additional_urls or None,
                     )
-                commit = get_commit(
-                    GitHub(get_best_robot_token(), per_page=100), pr_info.sha
-                )
+                commit = get_commit(gh, pr_info.sha)
                 post_commit_status(
                     commit,
                     job_report.status,
@@ -2147,12 +2151,49 @@ def main() -> int:
                 )
                 if not pr_info.is_merge_queue:
                     # in the merge queue mergeable status must be set only in FinishCheck (last job in wf)
-                    update_mergeable_check(
+                    mergeable_status = update_mergeable_check(
                         commit,
                         pr_info,
                         job_report.check_name or _get_ext_check_name(args.job_name),
                     )
 
+                    # Process upstream StatusNames.SYNC
+                    if (
+                        pr_info.head_ref.startswith(f"{SYNC_BRANCH_PREFIX}/pr/")
+                        and mergeable_status
+                        and GITHUB_REPOSITORY != GITHUB_UPSTREAM_REPOSITORY
+                    ):
+                        pr_number = int(pr_info.head_ref.split("/pr/", maxsplit=1)[1])
+                        upstream_repo = gh.get_repo(GITHUB_UPSTREAM_REPOSITORY)
+                        head_sha = upstream_repo.get_pull(pr_number).head.sha
+                        upstream_commit = upstream_repo.get_commit(head_sha)
+                        post_commit_status(
+                            upstream_commit,
+                            get_status(mergeable_status.state),
+                            "",  # let's won't expose any urls from cloud
+                            mergeable_status.description,
+                            StatusNames.SYNC,
+                        )
+                        trigger_mergeable_check(
+                            upstream_commit,
+                            get_commit_filtered_statuses(upstream_commit),
+                            True,
+                        )
+
+                        prepared_events = prepare_tests_results_for_clickhouse(
+                            pr_info,
+                            [],
+                            job_report.status,
+                            0,
+                            job_report.start_time,
+                            f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}",
+                            StatusNames.SYNC,
+                        )
+                        prepared_events[0]["test_context_raw"] = args.job_name
+                        ch_helper.insert_events_into(
+                            db="default", table="checks", events=prepared_events
+                        )
+
             print(f"Job report url: [{check_url}]")
             prepared_events = prepare_tests_results_for_clickhouse(
                 pr_info,

diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py
@@ -85,13 +85,13 @@ def post_commit_status(
     check_name: Optional[str] = None,
     pr_info: Optional[PRInfo] = None,
     dump_to_file: bool = False,
-) -> None:
+) -> CommitStatus:
     """The parameters are given in the same order as for commit.create_status,
     if an optional parameter `pr_info` is given, the `set_status_comment` functions
     is invoked to add or update the comment with statuses overview"""
     for i in range(RETRY):
         try:
-            commit.create_status(
+            commit_status = commit.create_status(
                 state=state,
                 target_url=report_url if report_url is not None else NotSet,
                 description=description if description is not None else NotSet,
@@ -128,6 +128,8 @@ def post_commit_status(
             pr_num=pr_info.number,
         ).dump_status()
 
+    return commit_status
+
 
 STATUS_ICON_MAP = defaultdict(
     str,
@@ -425,16 +427,23 @@ def set_mergeable_check(
     commit: Commit,
     description: str = "",
     state: StatusType = SUCCESS,
-) -> None:
-    commit.create_status(
-        context=StatusNames.MERGEABLE,
-        description=format_description(description),
-        state=state,
-        target_url=GITHUB_RUN_URL,
+    hide_url: bool = False,
+) -> CommitStatus:
+    report_url = GITHUB_RUN_URL
+    if hide_url:
+        report_url = ""
+    return post_commit_status(
+        commit,
+        state,
+        report_url,
+        format_description(description),
+        StatusNames.MERGEABLE,
     )
 
 
-def update_mergeable_check(commit: Commit, pr_info: PRInfo, check_name: str) -> None:
+def update_mergeable_check(
+    commit: Commit, pr_info: PRInfo, check_name: str
+) -> Optional[CommitStatus]:
     "check if the check_name in REQUIRED_CHECKS and then trigger update"
     not_run = (
         pr_info.labels.intersection({Labels.SKIP_MERGEABLE_CHECK, Labels.RELEASE})
@@ -445,15 +454,17 @@ def update_mergeable_check(commit: Commit, pr_info: PRInfo, check_name: str) ->
 
     if not_run:
         # Let's avoid unnecessary work
-        return
+        return None
 
     logging.info("Update Mergeable Check by %s", check_name)
 
     statuses = get_commit_filtered_statuses(commit)
-    trigger_mergeable_check(commit, statuses)
+    return trigger_mergeable_check(commit, statuses)
 
 
-def trigger_mergeable_check(commit: Commit, statuses: CommitStatuses) -> None:
+def trigger_mergeable_check(
+    commit: Commit, statuses: CommitStatuses, hide_url: bool = False
+) -> CommitStatus:
     """calculate and update StatusNames.MERGEABLE"""
     required_checks = [
         status for status in statuses if status.context in REQUIRED_CHECKS
@@ -486,4 +497,6 @@ def trigger_mergeable_check(commit: Commit, statuses: CommitStatuses) -> None:
     description = format_description(description)
 
     if mergeable_status is None or mergeable_status.description != description:
-        set_mergeable_check(commit, description, state)
+        return set_mergeable_check(commit, description, state, hide_url)
+
+    return mergeable_status
diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py
@@ -22,6 +22,9 @@
 GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")
 GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0")
 GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com")
+GITHUB_UPSTREAM_REPOSITORY = os.getenv(
+    "GITHUB_UPSTREAM_REPOSITORY", "ClickHouse/ClickHouse"
+)
 GITHUB_WORKSPACE = os.getenv("GITHUB_WORKSPACE", git_root)
 GITHUB_RUN_URL = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}"
 IMAGES_PATH = os.getenv("IMAGES_PATH", TEMP_PATH)

diff --git a/tests/ci/report.py b/tests/ci/report.py
@@ -49,6 +49,15 @@ def _state_rank(status: str) -> int:
         return 3
 
 
+def get_status(status: str) -> StatusType:
+    "function to get the StatusType for a status or ERROR"
+    try:
+        ind = STATUSES.index(status)  # type: ignore
+        return STATUSES[ind]
+    except ValueError:
+        return ERROR
+
+
 def get_worst_status(statuses: Iterable[str]) -> StatusType:
     worst_status = SUCCESS  # type: StatusType
     for status in statuses:

diff --git a/tests/ci/synchronizer_utils.py b/tests/ci/synchronizer_utils.py
@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+SYNC_BRANCH_PREFIX = "sync-upstream"
+SYNC_MASTER_BRANCH = f"{SYNC_BRANCH_PREFIX}/master"