Skip to content

Commit 3f417a3

Browse files
committed
merge: reconcile governance observability with latest master
2 parents 5b0a459 + aab453f commit 3f417a3

550 files changed

Lines changed: 21201 additions & 1041 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Container runtime files are consumed inside Linux images: a CRLF checkout on
18+
# Windows would break `sh entrypoint.sh` and Dockerfile inputs.
19+
shenyu-dist/**/docker/Dockerfile text eol=lf
20+
shenyu-dist/**/docker/*.sh text eol=lf
21+
script/*.sh text eol=lf
Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
"""Prepare and restore the Maven inputs for the selected E2E matrix."""
19+
20+
import json
21+
import os
22+
from pathlib import Path
23+
import sys
24+
import tarfile
25+
import xml.etree.ElementTree as ET
26+
from xml.sax.saxutils import escape
27+
28+
29+
def plan(directory, source):
30+
source = source.resolve(strict=True)
31+
examples = set()
32+
tests = set()
33+
for item in json.loads(os.environ.get("E2E_MATRIX", '{"include":[]}'))["include"]:
34+
examples.update(item["example_projects"].split(","))
35+
tests.add("shenyu-e2e-case/" + item["case"])
36+
for item in json.loads(os.environ.get("STORAGE_MATRIX", '{"include":[]}'))["include"]:
37+
tests.add("shenyu-e2e-case/" + item["case"])
38+
if not tests:
39+
raise ValueError("The selected E2E matrix is empty")
40+
(directory / "example-projects").write_text(",".join(sorted(examples)), encoding="utf-8")
41+
(directory / "test-projects").write_text(",".join(sorted(tests)), encoding="utf-8")
42+
# Both producer and consumer use the same repository IDs, preserving Maven's
43+
# repository-origin tracking. Central is restricted to releases; built
44+
# snapshots are read from this workflow's successful prerequisite build.
45+
repository = f"""<repository><id>e2e-built-inputs</id><url>{escape(source.as_uri())}</url>
46+
<releases><enabled>true</enabled><checksumPolicy>ignore</checksumPolicy></releases>
47+
<snapshots><enabled>true</enabled><checksumPolicy>ignore</checksumPolicy></snapshots></repository>"""
48+
plugin_repository = repository.replace("repository>", "pluginRepository>")
49+
(directory / "settings.xml").write_text(f"""<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0">
50+
<profiles><profile><id>e2e-inputs</id>
51+
<repositories>{repository}
52+
<repository><id>central</id><url>https://repo.maven.apache.org/maven2</url>
53+
<snapshots><enabled>false</enabled></snapshots></repository>
54+
</repositories>
55+
<pluginRepositories>{plugin_repository}</pluginRepositories>
56+
</profile></profiles>
57+
<activeProfiles><activeProfile>e2e-inputs</activeProfile></activeProfiles>
58+
</settings>
59+
""", encoding="utf-8")
60+
61+
62+
def build_tools(directory, pom):
63+
ns = {"m": "http://maven.apache.org/POM/4.0.0"}
64+
artifacts = set()
65+
for plugin in ET.parse(pom).findall(".//m:build/m:plugins/m:plugin", ns):
66+
if plugin.findtext("m:artifactId", namespaces=ns) == "protobuf-maven-plugin":
67+
for name in ("protocArtifact", "pluginArtifact"):
68+
artifacts.update(item.text for item in plugin.findall("m:configuration/m:" + name, ns))
69+
for compiler in plugin.findall("m:configuration/m:protocPlugins/m:protocPlugin", ns):
70+
artifacts.add(":".join(compiler.findtext("m:" + name, namespaces=ns)
71+
for name in ("groupId", "artifactId", "version")) + ":jar")
72+
(directory / "build-tools").write_text("".join(artifact + "\n" for artifact in sorted(artifacts)), encoding="utf-8")
73+
74+
75+
def dependency_inputs(directory):
76+
projects = []
77+
decoder = json.JSONDecoder()
78+
for name in ("example-dependencies.json", "test-dependencies.json"):
79+
path = directory / name
80+
if path.exists():
81+
content = path.read_text(encoding="utf-8").strip()
82+
while content:
83+
project, end = decoder.raw_decode(content)
84+
projects.append(project)
85+
content = content[end:].lstrip()
86+
reactor = {(project["groupId"], project["artifactId"], project["version"]) for project in projects}
87+
destination = directory / "dependencies"
88+
destination.mkdir()
89+
90+
def new_project(name):
91+
pom = ET.Element("project", {"xmlns": "http://maven.apache.org/POM/4.0.0"})
92+
for key, value in (("modelVersion", "4.0.0"), ("groupId", "org.apache.shenyu.e2e.inputs"),
93+
("artifactId", name), ("version", "1")):
94+
ET.SubElement(pom, key).text = value
95+
return pom
96+
97+
parent = new_project("dependencies")
98+
ET.SubElement(parent, "packaging").text = "pom"
99+
modules = ET.SubElement(parent, "modules")
100+
for index, project in enumerate(projects):
101+
artifacts = set()
102+
103+
def visit(node):
104+
coordinate = (node["groupId"], node["artifactId"], node["version"])
105+
if coordinate not in reactor:
106+
artifacts.add((*coordinate, node["type"], node.get("classifier", "")))
107+
for child in node.get("children", []):
108+
visit(child)
109+
110+
for child in project.get("children", []):
111+
visit(child)
112+
name = "project-" + str(index)
113+
pom = new_project(name)
114+
dependencies = ET.SubElement(pom, "dependencies")
115+
for group, artifact, version, kind, classifier in sorted(artifacts):
116+
dependency = ET.SubElement(dependencies, "dependency")
117+
for key, value in (("groupId", group), ("artifactId", artifact), ("version", version), ("type", kind)):
118+
ET.SubElement(dependency, key).text = value
119+
if classifier:
120+
ET.SubElement(dependency, "classifier").text = classifier
121+
# Versions were already mediated by Maven's dependency tree. Do not
122+
# resolve transitives again or reintroduce unbuilt reactor modules.
123+
exclusion = ET.SubElement(ET.SubElement(dependency, "exclusions"), "exclusion")
124+
ET.SubElement(exclusion, "groupId").text = "*"
125+
ET.SubElement(exclusion, "artifactId").text = "*"
126+
module = destination / name
127+
module.mkdir()
128+
ET.ElementTree(pom).write(module / "pom.xml", encoding="utf-8", xml_declaration=True)
129+
ET.SubElement(modules, "module").text = name
130+
# Keep each project's versions separate: a single union POM would mediate
131+
# conflicting versions again and discard inputs needed by other matrix jobs.
132+
ET.ElementTree(parent).write(destination / "pom.xml", encoding="utf-8", xml_declaration=True)
133+
134+
135+
def pack(directory, maven_user_home):
136+
repository = directory / "repository"
137+
version = (directory / "maven-version.txt").read_text(encoding="utf-8")
138+
maven_home = Path(next(
139+
line.removeprefix("Maven home:").strip()
140+
for line in version.splitlines() if line.startswith("Maven home:")))
141+
distribution = maven_home.parent
142+
relative = distribution.relative_to(maven_user_home.resolve())
143+
if relative.parts[:2] != ("wrapper", "dists") or not (maven_home / "bin/mvn").is_file():
144+
raise ValueError("Unexpected Maven wrapper distribution: " + str(maven_home))
145+
payload = [
146+
(repository, Path("repository")),
147+
(distribution, Path("maven-user-home") / relative),
148+
(Path(".mvn/wrapper/maven-wrapper.jar"), Path("wrapper/maven-wrapper.jar")),
149+
]
150+
archive = directory / "repository.tar.gz"
151+
with tarfile.open(archive, "w:gz") as output:
152+
for source, target in payload:
153+
output.add(source, arcname=str(target))
154+
print("Prepared the selected E2E Maven repository and wrapper")
155+
156+
157+
def restore(directory, destination):
158+
archive = directory / "repository.tar.gz"
159+
if destination.exists():
160+
raise ValueError("E2E Maven repository destination must not exist: " + str(destination))
161+
destination.mkdir(parents=True)
162+
with tarfile.open(archive, "r:gz") as source:
163+
source.extractall(destination, filter="data")
164+
repository = destination / "repository"
165+
namespace = "http://maven.apache.org/SETTINGS/1.0.0"
166+
ET.register_namespace("", namespace)
167+
settings = ET.parse(directory / "settings.xml")
168+
ns = {"s": namespace}
169+
repositories = settings.findall(".//s:repository", ns) + settings.findall(".//s:pluginRepository", ns)
170+
for item in repositories:
171+
if item.find("s:id", ns).text == "e2e-built-inputs":
172+
item.find("s:url", ns).text = repository.as_uri()
173+
ET.SubElement(settings.getroot(), f"{{{namespace}}}offline").text = "true"
174+
ET.SubElement(settings.getroot(), f"{{{namespace}}}localRepository").text = str(repository)
175+
settings.write(destination / "settings.xml", encoding="utf-8", xml_declaration=True)
176+
print("Restored E2E Maven inputs with offline settings")
177+
178+
179+
if __name__ == "__main__":
180+
command, artifact = sys.argv[1:3]
181+
directory = Path(artifact).resolve()
182+
if command == "plan":
183+
plan(directory, Path(sys.argv[3]))
184+
elif command == "build-tools":
185+
build_tools(directory, Path(sys.argv[3]))
186+
elif command == "dependencies":
187+
dependency_inputs(directory)
188+
elif command == "pack":
189+
pack(directory, Path(sys.argv[3]))
190+
elif command == "restore":
191+
restore(directory, Path(sys.argv[3]).resolve())
192+
else:
193+
raise ValueError("Unknown E2E Maven repository command: " + command)

‎.github/scripts/pr-comment-commands.cjs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,9 @@ async function retestPullRequest({context, github, core, pullRequest}) {
224224
run_id: run.id,
225225
per_page: 100,
226226
});
227-
return !artifacts.some(artifact => artifact.name === 'shenyu-images' && !artifact.expired);
227+
// Failed-job reruns preserve the producer, so both E2E inputs must still exist.
228+
return !['shenyu-images', 'shenyu-local-maven-repository'].every(name =>
229+
artifacts.some(artifact => artifact.name === name && !artifact.expired));
228230
};
229231
for (const run of latest.values()) {
230232
if (!isFailed(run)) {
@@ -244,9 +246,9 @@ async function retestPullRequest({context, github, core, pullRequest}) {
244246
core.info(`Run ${run.id} has already changed; skipping it.`);
245247
continue;
246248
}
247-
if (await needsFullRerun(run)) {
249+
if (await needsFullRerun(currentRun)) {
248250
await github.rest.actions.reRunWorkflow({...context.repo, run_id: run.id});
249-
core.info(`Requested full rerun, the e2e image artifact is gone: ${run.html_url}`);
251+
core.info(`Requested full rerun, an E2E artifact is unavailable: ${run.html_url}`);
250252
} else {
251253
await github.rest.actions.reRunWorkflowFailedJobs({...context.repo, run_id: run.id});
252254
core.info(`Requested failed-job rerun: ${run.html_url}`);

0 commit comments

Comments
 (0)