Skip to content

Commit 2c981cc

Browse files
Integrate landed diagnostics into context metadata
2 parents 7be4577 + b03f556 commit 2c981cc

43 files changed

Lines changed: 1797 additions & 385 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
"""Reuse Cargo work only from an identical canonical CodeQL push."""
2+
3+
import hashlib
4+
import importlib.util
5+
import json
6+
import os
7+
import shutil
8+
import subprocess
9+
import sys
10+
import tarfile
11+
import tempfile
12+
from datetime import datetime
13+
from pathlib import Path, PurePosixPath
14+
from zipfile import ZipFile
15+
16+
NAME = "codeql-cargo-cache-Linux-X64"
17+
WORKFLOW = ".github/workflows/codeql.yml"
18+
BRANCH = "feat/codex-harness"
19+
ROOT = Path.cwd().resolve()
20+
NATIVE = ROOT / "packages/codex"
21+
ROOTS = {
22+
"upstream": NATIVE / ".upstream",
23+
"registry": Path(os.environ.get("CARGO_HOME", str(Path.home() / ".cargo"))) / "registry",
24+
"git": Path(os.environ.get("CARGO_HOME", str(Path.home() / ".cargo"))) / "git",
25+
"target": Path(
26+
os.environ.get("CODEQL_EXTRACTOR_RUST_OPTION_CARGO_TARGET_DIR", "/tmp/codeql-cargo-target")
27+
),
28+
}
29+
30+
31+
def run(*args):
32+
return subprocess.check_output(args, text=True, timeout=120).strip()
33+
34+
35+
def api(path):
36+
return json.loads(run("gh", "api", f"repos/{os.environ['GITHUB_REPOSITORY']}/{path}"))
37+
38+
39+
def inputs(ref):
40+
entries = run("git", "ls-tree", "-rz", "--full-tree", ref).split("\0")
41+
selected = []
42+
for entry in entries:
43+
if not entry:
44+
continue
45+
_, path = entry.split("\t", 1)
46+
if (
47+
path.startswith("packages/codex/rust/")
48+
and path.endswith(".rs")
49+
and Path(path).name != "build.rs"
50+
):
51+
continue
52+
if (
53+
path.startswith(("packages/codex/", ".github/scripts/", ".github/actions/"))
54+
or path == WORKFLOW
55+
or path in {"Makefile", "pyproject.toml", "uv.lock"}
56+
or path.endswith((".rs", "Cargo.toml", "Cargo.lock", "rust-toolchain.toml"))
57+
or "/.cargo/" in path
58+
or path.startswith(".cargo/")
59+
):
60+
selected.append(entry)
61+
return hashlib.sha256("\0".join(selected).encode()).hexdigest()
62+
63+
64+
def identity(ref="HEAD"):
65+
return {
66+
"inputs": inputs(ref),
67+
"rustc": run("rustc", "-Vv"),
68+
"codeql": os.environ["CODEQL_CLI_VERSION"],
69+
"platform": [os.environ["RUNNER_OS"], os.environ["RUNNER_ARCH"]],
70+
"environment": {
71+
key: os.environ.get(key, "")
72+
for key in (
73+
"ImageOS",
74+
"ImageVersion",
75+
"RUSTFLAGS",
76+
"RUSTDOCFLAGS",
77+
"CARGO_BUILD_TARGET",
78+
"CC",
79+
"CXX",
80+
)
81+
},
82+
"workspace": str(ROOT),
83+
"roots": {key: str(value) for key, value in ROOTS.items()},
84+
}
85+
86+
87+
def verify_sources(root=NATIVE):
88+
sys.path.insert(0, str(NATIVE / "scripts"))
89+
from source_integrity import verify_pristine_source, verify_rmcp_source
90+
91+
verify_pristine_source(root)
92+
verify_rmcp_source(root)
93+
94+
95+
def load_downloader():
96+
spec = importlib.util.spec_from_file_location(
97+
"native_snapshot", ROOT / ".github/scripts/download-native-snapshot.py"
98+
)
99+
module = importlib.util.module_from_spec(spec)
100+
spec.loader.exec_module(module)
101+
return module.download_archive
102+
103+
104+
def unpack(archive):
105+
"""Validate the entire archive before materializing any cached files."""
106+
with tarfile.open(archive) as contents:
107+
members = contents.getmembers()
108+
names = set()
109+
if len(members) > 400_000 or sum(item.size for item in members) > 30 * 1024**3:
110+
raise ValueError("CodeQL Cargo cache exceeds extraction limits")
111+
for item in members:
112+
parts = PurePosixPath(item.name).parts
113+
if (
114+
not parts
115+
or parts[0] not in ROOTS
116+
or item.name != "/".join(parts)
117+
or any(part in {".", ".."} for part in parts)
118+
or "\\" in item.name
119+
or item.name in names
120+
or not (item.isfile() or item.isdir() or item.issym())
121+
):
122+
raise ValueError("Unexpected CodeQL Cargo cache member")
123+
names.add(item.name)
124+
if item.issym():
125+
destination = ROOTS[parts[0]].joinpath(*parts[1:])
126+
if parts[0] != "upstream" or not (
127+
destination.parent / item.linkname
128+
).resolve().is_relative_to(ROOTS["upstream"].resolve()):
129+
raise ValueError("CodeQL Cargo cache link escapes native sources")
130+
with tempfile.TemporaryDirectory() as temporary:
131+
contents.extractall(temporary, filter="data")
132+
# Preserve the archived source mtimes alongside Cargo fingerprints.
133+
staged = Path(temporary) / "native"
134+
staged.mkdir()
135+
(staged / ".upstream").symlink_to(
136+
Path(temporary) / "upstream", target_is_directory=True
137+
)
138+
for name in ("CODEX_REVISION", "Cargo.toml", "patches", "rmcp-patches"):
139+
source = NATIVE / name
140+
if source.is_dir():
141+
shutil.copytree(source, staged / name)
142+
else:
143+
shutil.copy2(source, staged / name)
144+
verify_sources(staged)
145+
for key, destination in ROOTS.items():
146+
source = Path(temporary) / key
147+
if not source.is_dir() or (key != "upstream" and destination.exists()):
148+
raise ValueError("CodeQL Cargo cache destination is not fresh")
149+
if ROOTS["upstream"].exists():
150+
shutil.rmtree(ROOTS["upstream"])
151+
for key, destination in ROOTS.items():
152+
destination.parent.mkdir(parents=True, exist_ok=True)
153+
shutil.copytree(
154+
Path(temporary) / key, destination, symlinks=True, copy_function=shutil.copy2
155+
)
156+
verify_sources()
157+
158+
159+
def restore():
160+
current = identity()
161+
workflow = api("actions/workflows/codeql.yml")
162+
runs = api(
163+
f"actions/workflows/codeql.yml/runs?branch={BRANCH}&event=push&status=success&per_page=5"
164+
)["workflow_runs"]
165+
for candidate in runs:
166+
if (
167+
candidate["workflow_id"] != workflow["id"]
168+
or candidate["event"] != "push"
169+
or candidate["head_branch"] != BRANCH
170+
or candidate["head_repository"]["full_name"] != "openai/openai-agents-python"
171+
or candidate["path"] != WORKFLOW
172+
or candidate["conclusion"] != "success"
173+
or candidate["run_attempt"] != 1
174+
):
175+
continue
176+
run_id = str(candidate["id"])
177+
run("git", "fetch", "--no-tags", "--depth=1", "origin", candidate["head_sha"])
178+
if inputs(candidate["head_sha"]) != current["inputs"]:
179+
continue
180+
jobs = api(f"actions/runs/{run_id}/jobs?per_page=100")
181+
if jobs["total_count"] > 100:
182+
raise ValueError("Incomplete CodeQL job listing")
183+
rust = [
184+
job
185+
for job in jobs["jobs"]
186+
if job["name"] == "CodeQL (rust)" and job["conclusion"] == "success"
187+
]
188+
if len(rust) != 1 or not any(
189+
step["name"] == "Publish CodeQL Cargo cache" and step["conclusion"] == "success"
190+
for step in rust[0]["steps"]
191+
):
192+
continue
193+
result = api(f"actions/runs/{run_id}/artifacts?per_page=100")
194+
if result["total_count"] > 100:
195+
raise ValueError("Incomplete CodeQL artifact listing")
196+
artifacts = [
197+
item for item in result["artifacts"] if item["name"] == NAME and not item["expired"]
198+
]
199+
if not artifacts:
200+
continue
201+
if len(artifacts) != 1:
202+
raise ValueError("Ambiguous CodeQL Cargo cache publication")
203+
artifact = artifacts[0]
204+
(publication,) = [
205+
step for step in rust[0]["steps"] if step["name"] == "Publish CodeQL Cargo cache"
206+
]
207+
times = [
208+
datetime.fromisoformat(value.replace("Z", "+00:00"))
209+
for value in (
210+
publication["started_at"],
211+
artifact["created_at"],
212+
publication["completed_at"],
213+
)
214+
]
215+
if not times[0] <= times[1] <= times[2]:
216+
raise ValueError("CodeQL Cargo cache was created outside the Rust publication step")
217+
with tempfile.TemporaryFile() as downloaded, tempfile.TemporaryDirectory() as temporary:
218+
load_downloader()(artifact["id"], artifact["digest"], run_id, NAME, downloaded)
219+
with ZipFile(downloaded) as bundle:
220+
if sorted(bundle.namelist()) != ["cargo.tar", "identity.json"]:
221+
raise ValueError("Unexpected CodeQL Cargo cache inventory")
222+
captured = json.loads(bundle.read("identity.json"))
223+
if captured != current:
224+
print("Selected CodeQL Cargo cache identity differs; extracting cold")
225+
return
226+
archive = Path(temporary) / "cargo.tar"
227+
with bundle.open("cargo.tar") as source, archive.open("wb") as target:
228+
shutil.copyfileobj(source, target)
229+
unpack(archive)
230+
print(f"Restored verified CodeQL Cargo cache from run {run_id}")
231+
return
232+
print("No equivalent canonical CodeQL Cargo cache; extracting cold")
233+
234+
235+
def pack():
236+
verify_sources()
237+
# Package-scoped Cargo clean defaults to dev, so clean every observed layout.
238+
target = ROOTS["target"]
239+
fingerprints = sorted(target.glob("*/.fingerprint")) + sorted(target.glob("*/*/.fingerprint"))
240+
for directory in fingerprints:
241+
profile = directory.parent
242+
options = ["--profile", "dev" if profile.name == "debug" else profile.name]
243+
if profile.parent != target:
244+
options += ["--target", profile.parent.name]
245+
subprocess.run(
246+
[
247+
"cargo",
248+
"clean",
249+
"--frozen",
250+
"--manifest-path",
251+
str(NATIVE / "Cargo.toml"),
252+
"--target-dir",
253+
str(target),
254+
"--package",
255+
"codex-agents-python",
256+
*options,
257+
],
258+
check=True,
259+
)
260+
destination = Path(os.environ["RUNNER_TEMP"]) / "codeql-cargo-publication"
261+
destination.mkdir()
262+
(destination / "identity.json").write_text(json.dumps(identity(), sort_keys=True))
263+
with tarfile.open(destination / "cargo.tar", "w") as archive:
264+
for key, source in ROOTS.items():
265+
source.mkdir(parents=True, exist_ok=True)
266+
for path in [source, *sorted(source.rglob("*"))]:
267+
if path.is_symlink() and key != "upstream":
268+
raise ValueError("Unexpected link in Cargo dependency cache")
269+
member = archive.gettarinfo(
270+
str(path), arcname=str(Path(key) / path.relative_to(source))
271+
)
272+
# Store hard-linked regular files independently.
273+
if member.islnk():
274+
member.type, member.linkname, member.size = (
275+
tarfile.REGTYPE,
276+
"",
277+
path.stat().st_size,
278+
)
279+
if member.isfile():
280+
with path.open("rb") as data:
281+
archive.addfile(member, data)
282+
elif member.isdir() or member.issym():
283+
archive.addfile(member)
284+
else:
285+
raise ValueError("Unsupported Cargo cache file")
286+
287+
288+
if __name__ == "__main__":
289+
if sys.argv[1] == "restore":
290+
restore()
291+
elif sys.argv[1] == "pack":
292+
pack()
293+
else:
294+
raise SystemExit("Expected restore or pack")

0 commit comments

Comments
 (0)