Skip to content

Commit 85d468a

Browse files
committed
Merge latest master into PR #7569
63 Wasm bridge and loader tests passed; Checkstyle and RAT passed. Signed-off-by: redyyang <[email protected]>
2 parents 4787364 + d7e8282 commit 85d468a

177 files changed

Lines changed: 13674 additions & 1423 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/scripts/actions-timings.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#!/usr/bin/env python3
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Export per-job queue, preparation and verification seconds using the GitHub API.
18+
19+
Usage: python3 .github/scripts/actions-timings.py --repo apache/shenyu --limit 50 > timings.csv
20+
Queue includes dependency wait; runner_queue_seconds shows the API-observable runner wait.
21+
Verification is the duration of test/check steps, including compilation within those steps.
22+
Image builds, tool installation, deployment, cache handling and post steps are preparation.
23+
Failed, skipped and cancelled runs remain visible and must not be averaged with successful runs.
24+
"""
25+
26+
import argparse
27+
import concurrent.futures
28+
import csv
29+
import datetime
30+
import json
31+
import re
32+
import subprocess
33+
import sys
34+
import time
35+
36+
VERIFY = re.compile(
37+
r"^(?:Run (?:test|Maven tests|E2E Tests|Java 21 compatibility tests|Apache RAT)|"
38+
r"Build with Maven$|Replay registration contract|Check resolved registry|"
39+
r"Check Agent|Check License|Check WAT|Verify list/detail|Run native fresh|"
40+
r"Check harness|Build disposable Admin|Perform CodeQL|Test |Check test result|"
41+
r"Cluster Test|Require every|checking job status|python3 .*test-)", re.IGNORECASE)
42+
43+
44+
def api(path):
45+
"""Retry transient API failures once, respecting the repository's backoff policy."""
46+
for attempt in range(2):
47+
result = subprocess.run(["gh", "api", path], text=True, capture_output=True)
48+
if result.returncode == 0:
49+
return json.loads(result.stdout)
50+
if attempt == 0 and re.search(r"429|5\d\d|timeout|timed out", result.stderr, re.IGNORECASE):
51+
time.sleep(20 if "429" in result.stderr else 2)
52+
continue
53+
raise RuntimeError(result.stderr.strip())
54+
55+
56+
def seconds(start, end):
57+
"""Incomplete/skipped timestamps have no measurable duration."""
58+
if not start or not end:
59+
return 0
60+
return max(0, (datetime.datetime.fromisoformat(end.replace("Z", "+00:00"))
61+
- datetime.datetime.fromisoformat(start.replace("Z", "+00:00"))).total_seconds())
62+
63+
64+
def row(run, job, repo):
65+
"""Partition the job's lifetime, retaining skipped jobs as zero-duration rows."""
66+
elapsed = seconds(job.get("started_at"), job.get("completed_at"))
67+
verification = 0
68+
for step in job.get("steps", []):
69+
if step.get("conclusion") == "skipped":
70+
continue
71+
name = step["name"]
72+
# Build-only jobs use this same label; their tests are disabled explicitly.
73+
if name == "Build with Maven" and not (run["name"] == "ci" and job["name"] == "pr_build"):
74+
continue
75+
if VERIFY.search(name):
76+
verification += seconds(step.get("started_at"), step.get("completed_at"))
77+
skipped = job.get("conclusion") == "skipped"
78+
queue = 0 if skipped else seconds(run["created_at"], job.get("started_at"))
79+
runner_queue = 0 if skipped else seconds(job.get("created_at"), job.get("started_at"))
80+
return {
81+
"repository": repo, "run_id": run["id"], "attempt": run["run_attempt"],
82+
"workflow": run["name"], "event": run["event"], "sha": run["head_sha"],
83+
"job_id": job["id"], "job": job["name"], "status": job["status"],
84+
"conclusion": job.get("conclusion"), "queue_seconds": queue,
85+
"runner_queue_seconds": runner_queue, "preparation_seconds": max(0, elapsed - verification),
86+
"verification_seconds": verification, "url": job["html_url"],
87+
}
88+
89+
90+
def collect(repo, run):
91+
jobs = []
92+
page = 1
93+
while True:
94+
response = api(f"repos/{repo}/actions/runs/{run['id']}/jobs?per_page=100&page={page}")
95+
jobs.extend(response["jobs"])
96+
if len(jobs) >= response["total_count"]:
97+
return [row(run, job, repo) for job in jobs]
98+
page += 1
99+
100+
101+
def main():
102+
parser = argparse.ArgumentParser(description=__doc__)
103+
parser.add_argument("--repo", default="apache/shenyu")
104+
parser.add_argument("--limit", type=int, default=50, choices=range(1, 101))
105+
parser.add_argument("--run", action="append", type=int, help="Inspect specific run IDs instead of recent runs")
106+
args = parser.parse_args()
107+
if args.run:
108+
runs = [api(f"repos/{args.repo}/actions/runs/{run_id}") for run_id in args.run]
109+
else:
110+
runs = api(f"repos/{args.repo}/actions/runs?per_page={args.limit}")["workflow_runs"]
111+
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
112+
results = list(pool.map(lambda run: collect(args.repo, run), runs))
113+
rows = [item for group in results for item in group]
114+
if rows:
115+
writer = csv.DictWriter(sys.stdout, fieldnames=list(rows[0]))
116+
writer.writeheader()
117+
writer.writerows(rows)
118+
119+
120+
if __name__ == "__main__":
121+
main()

‎.github/scripts/expand-ci-module-roots.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,18 @@ set -euo pipefail
1919

2020
group_roots="${GROUP_ROOTS:-}"
2121
modules=()
22+
excludes=()
2223

2324
add_module() {
2425
local module="$1"
2526
local existing
27+
local excluded
28+
29+
for excluded in ${excludes[@]+"${excludes[@]}"}; do
30+
if [[ "${module}" == "${excluded}" || "${module}" == "${excluded}/"* ]]; then
31+
return
32+
fi
33+
done
2634

2735
for existing in "${modules[@]}"; do
2836
if [[ "${existing}" == "${module}" ]]; then
@@ -34,8 +42,20 @@ add_module() {
3442
}
3543

3644
IFS=',' read -ra roots <<< "${group_roots}"
45+
# Move a slow subtree into its own shard without duplicating the remaining roots.
46+
for root in "${roots[@]}"; do
47+
if [[ "${root}" == \!* ]]; then
48+
excluded="${root#\!}"
49+
if [[ ! -f "${excluded}/pom.xml" ]]; then
50+
echo "Excluded Maven module root is missing: ${excluded}" >&2
51+
exit 1
52+
fi
53+
excludes+=("${excluded}")
54+
fi
55+
done
3756
for root in "${roots[@]}"; do
3857
[[ -n "${root}" ]] || continue
58+
[[ "${root}" != \!* ]] || continue
3959

4060
if [[ ! -f "${root}/pom.xml" ]]; then
4161
echo "Maven module root is missing: ${root}" >&2

‎.github/scripts/native-sql-matrix.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,16 @@ def verify_index_rows(rows, postgres=False):
108108
indexes.setdefault((table, name), []).append((int(position), column))
109109
if not postgres:
110110
indexes = {key: [column for _, column in sorted(values)] for key, values in indexes.items()}
111-
for table, column in list(REQUIRED.items()) + [("permission", "object_id"), ("permission", "resource_id")]:
111+
for table, column in list(REQUIRED.items()) + [("permission", "object_id"), ("permission", "resource_id"),
112+
("registration_fencing_target", "domain_id")]:
112113
if not any(t == table and columns and columns[0] == column for (t, _), columns in indexes.items()):
113114
raise AssertionError(f"Missing leading-column index: {table}({column})")
114115
for table, name, column in [("selector", "idx_selector_plugin_id", "plugin_id"),
115116
("permission", "idx_permission_resource_id", "resource_id")]:
116117
if indexes.get((table, name)) != [column]:
117118
raise AssertionError(f"Wrong index columns: {name}")
119+
if indexes.get(("registration_fencing_target", "idx_fencing_domain_session")) != ["domain_id", "session_id"]:
120+
raise AssertionError("Wrong index columns: idx_fencing_domain_session")
118121
return indexes
119122

120123

‎.github/scripts/resolve-ci-modules.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ is_ignored_change() {
8484
local file="$1"
8585

8686
case "${file}" in
87+
.github/workflows/ci.yml|.github/scripts/*|actions/*)
88+
return 1
89+
;;
8790
# The opt-in reactor is packaged and smoke-tested by minimal-http.yml.
8891
shenyu-bootstrap-http/*|script/performance/minimal-http/*)
8992
return 0
@@ -109,7 +112,7 @@ while IFS= read -r file; do
109112
has_code_changes=true
110113

111114
case "${file}" in
112-
pom.xml|mvnw|mvnw.cmd|.mvn/*|actions/*)
115+
pom.xml|mvnw|mvnw.cmd|.mvn/*|actions/*|.github/workflows/ci.yml|.github/scripts/*)
113116
full_build_required=true
114117
;;
115118
esac
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/usr/bin/env python3
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Select every example image consumed by an integration case, before building."""
18+
19+
import re
20+
import sys
21+
import xml.etree.ElementTree as ET
22+
from pathlib import Path
23+
24+
ROOT = Path(__file__).resolve().parents[2]
25+
NS = {"m": "http://maven.apache.org/POM/4.0.0"}
26+
IMAGE = re.compile(r'''(?:image:\s*|kind load docker-image\s+)["']?(?:apache/)?(shenyu-examples?-[\w-]+):''')
27+
28+
29+
def resolve(case, root=ROOT):
30+
"""Fail closed when a new deployment has no identifiable example image."""
31+
if not re.fullmatch(r"shenyu-integrated-test-[a-z0-9-]+", case):
32+
raise ValueError(f"Invalid integration case: {case}")
33+
directory = root / "shenyu-integrated-test" / case
34+
if not (directory / "pom.xml").is_file():
35+
raise ValueError(f"Integration case does not exist: {case}")
36+
images = set()
37+
for path in directory.rglob("*"):
38+
if "target" not in path.parts and path.suffix in (".sh", ".yaml", ".yml"):
39+
images.update(IMAGE.findall(path.read_text()))
40+
if not images:
41+
raise ValueError(f"No example images found for {case}; inspect its deployment before enabling CI")
42+
artifacts = {
43+
ET.parse(path).getroot().findtext("m:artifactId", namespaces=NS)
44+
for path in (root / "shenyu-examples").rglob("pom.xml")
45+
if "target" not in path.parts
46+
}
47+
if images - artifacts:
48+
raise ValueError(f"Example images without Maven modules: {sorted(images - artifacts)}")
49+
return ",".join(":" + image for image in sorted(images))
50+
51+
52+
if __name__ == "__main__":
53+
print(resolve(sys.argv[1]))

‎.github/scripts/resolve-test-case-matrix-test.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,4 +278,8 @@ for infrastructure in \
278278
assert_file_output "e2e" "${infrastructure}" "run_e2e_cases" "true"
279279
done
280280

281+
282+
assert_file_output "k8s-examples-http" ".github/workflows/k8s-examples-http.yml" "run_k8s_examples" "true"
283+
assert_file_output "k8s-examples-http" "actions/setup-java-with-retry/action.yml" "run_k8s_examples" "true"
284+
assert_file_output "e2e" ".github/scripts/select-e2e-sync.sh" "run_e2e_cases" "true"
281285
echo "CI test routing tests passed"

‎.github/scripts/resolve-test-case-matrix.sh‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ resolve_k8s_change() {
181181
return 0
182182
fi
183183

184+
184185
if [[ "${mode}" == "k8s-examples-http" ]]; then
185186
case "${file}" in
186187
pom.xml|mvnw|mvnw.cmd|.mvn/*|\
@@ -312,6 +313,8 @@ map_direct_case_path() {
312313
;;
313314
esac
314315
fi
316+
317+
315318
}
316319

317320
map_domain_path() {
@@ -426,6 +429,11 @@ while IFS= read -r file; do
426429

427430
if [[ "${mode}" == "e2e" ]]; then
428431
case "${file}" in
432+
.github/scripts/select-e2e-sync.sh)
433+
add_admin_register_e2e_cases
434+
add_unique e2e_cases "e2e-logging-rocketmq-compose"
435+
continue
436+
;;
429437
.github/workflows/e2e-k8s.yml|.github/scripts/*e2e-maven-repository*|.github/scripts/resolve-test-case-matrix.sh|.github/scripts/resolve-test-case-matrix-test.sh)
430438
full_required=true
431439
continue
@@ -435,13 +443,32 @@ while IFS= read -r file; do
435443

436444
if [[ "${mode}" == "k8s-ingress" ]]; then
437445
case "${file}" in
438-
.github/workflows/integrated-test-k8s-ingress.yml)
446+
.github/workflows/integrated-test-k8s-ingress.yml|.github/scripts/resolve-example-projects.py|actions/*)
439447
add_k8s_ingress_all
440448
continue
441449
;;
442450
esac
443451
fi
444452

453+
# Changes to the workflow or shared preparation code must exercise the matrix.
454+
if [[ "${mode}" == "integration" ]]; then
455+
case "${file}" in
456+
.github/workflows/integrated-test.yml|.github/scripts/resolve-example-projects.py|.github/scripts/resolve-test-case-matrix*|actions/*)
457+
full_required=true
458+
continue
459+
;;
460+
esac
461+
fi
462+
463+
if [[ "${mode}" == "k8s-examples-http" ]]; then
464+
case "${file}" in
465+
.github/workflows/k8s-examples-http.yml|.github/scripts/install-k3s*|.github/scripts/free-disk-space*|actions/*)
466+
run_k8s_examples=true
467+
continue
468+
;;
469+
esac
470+
fi
471+
445472
if is_ignored_change "${file}"; then
446473
continue
447474
fi

‎.github/scripts/restore-shenyu-local-repo.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@
1818
set -euo pipefail
1919

2020
artifact_dir="${1:-/tmp/shenyu-m2}"
21-
target_dir="${HOME}/.m2/repository/org/apache"
21+
target_dir="${2:-${HOME}/.m2/repository}/org/apache"
22+
23+
if [[ -f "${artifact_dir}/repository.tar.gz" ]]; then
24+
tar -xzf "${artifact_dir}/repository.tar.gz" -C "${artifact_dir}"
25+
fi
2226

2327
if [[ ! -d "${artifact_dir}/org/apache/shenyu" ]]; then
2428
echo "ShenYu local repository artifact is missing: ${artifact_dir}/org/apache/shenyu" >&2
@@ -27,7 +31,9 @@ fi
2731

2832
mkdir -p "${target_dir}"
2933
if [[ -d "${target_dir}/shenyu" ]]; then
30-
mv "${target_dir}/shenyu" "${target_dir}/shenyu.backup.${GITHUB_RUN_ID:-$$}"
34+
# Backups under ~/.m2/repository grow every saved dependency cache indefinitely.
35+
backup_dir="$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/shenyu-m2-previous.XXXXXX")"
36+
mv "${target_dir}/shenyu" "${backup_dir}/shenyu"
3137
fi
3238
cp -R "${artifact_dir}/org/apache/shenyu" "${target_dir}/shenyu"
3339

‎.github/scripts/select-e2e-sync.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
# shellcheck disable=SC2034
19+
# SYNC_ARRAY is consumed by the sourcing protocol runners; manual runs keep all channels.
20+
SYNC_ARRAY=("websocket" "http" "zookeeper")
21+
if [[ -n "${SHENYU_E2E_SYNC:-}" ]]; then
22+
case "${SHENYU_E2E_SYNC}" in
23+
websocket|http|zookeeper) SYNC_ARRAY=("${SHENYU_E2E_SYNC}") ;;
24+
*)
25+
echo "Unsupported E2E sync channel: ${SHENYU_E2E_SYNC}" >&2
26+
exit 1
27+
;;
28+
esac
29+
fi

0 commit comments

Comments
 (0)