|
| 1 | +"""Reuse Cargo work only from an identical canonical CodeQL push.""" |
| 2 | + |
| 3 | +import hashlib |
| 4 | +import importlib.util |
| 5 | +import json |
| 6 | +import os |
| 7 | +import shutil |
| 8 | +import subprocess |
| 9 | +import sys |
| 10 | +import tarfile |
| 11 | +import tempfile |
| 12 | +from datetime import datetime |
| 13 | +from pathlib import Path, PurePosixPath |
| 14 | +from zipfile import ZipFile |
| 15 | + |
| 16 | +NAME = "codeql-cargo-cache-Linux-X64" |
| 17 | +WORKFLOW = ".github/workflows/codeql.yml" |
| 18 | +BRANCH = "feat/codex-harness" |
| 19 | +ROOT = Path.cwd().resolve() |
| 20 | +NATIVE = ROOT / "packages/codex" |
| 21 | +ROOTS = { |
| 22 | + "upstream": NATIVE / ".upstream", |
| 23 | + "registry": Path(os.environ.get("CARGO_HOME", str(Path.home() / ".cargo"))) / "registry", |
| 24 | + "git": Path(os.environ.get("CARGO_HOME", str(Path.home() / ".cargo"))) / "git", |
| 25 | + "target": Path( |
| 26 | + os.environ.get("CODEQL_EXTRACTOR_RUST_OPTION_CARGO_TARGET_DIR", "/tmp/codeql-cargo-target") |
| 27 | + ), |
| 28 | +} |
| 29 | + |
| 30 | + |
| 31 | +def run(*args): |
| 32 | + return subprocess.check_output(args, text=True, timeout=120).strip() |
| 33 | + |
| 34 | + |
| 35 | +def api(path): |
| 36 | + return json.loads(run("gh", "api", f"repos/{os.environ['GITHUB_REPOSITORY']}/{path}")) |
| 37 | + |
| 38 | + |
| 39 | +def inputs(ref): |
| 40 | + entries = run("git", "ls-tree", "-rz", "--full-tree", ref).split("\0") |
| 41 | + selected = [] |
| 42 | + for entry in entries: |
| 43 | + if not entry: |
| 44 | + continue |
| 45 | + _, path = entry.split("\t", 1) |
| 46 | + if ( |
| 47 | + path.startswith("packages/codex/rust/") |
| 48 | + and path.endswith(".rs") |
| 49 | + and Path(path).name != "build.rs" |
| 50 | + ): |
| 51 | + continue |
| 52 | + if ( |
| 53 | + path.startswith(("packages/codex/", ".github/scripts/", ".github/actions/")) |
| 54 | + or path == WORKFLOW |
| 55 | + or path in {"Makefile", "pyproject.toml", "uv.lock"} |
| 56 | + or path.endswith((".rs", "Cargo.toml", "Cargo.lock", "rust-toolchain.toml")) |
| 57 | + or "/.cargo/" in path |
| 58 | + or path.startswith(".cargo/") |
| 59 | + ): |
| 60 | + selected.append(entry) |
| 61 | + return hashlib.sha256("\0".join(selected).encode()).hexdigest() |
| 62 | + |
| 63 | + |
| 64 | +def identity(ref="HEAD"): |
| 65 | + return { |
| 66 | + "inputs": inputs(ref), |
| 67 | + "rustc": run("rustc", "-Vv"), |
| 68 | + "codeql": os.environ["CODEQL_CLI_VERSION"], |
| 69 | + "platform": [os.environ["RUNNER_OS"], os.environ["RUNNER_ARCH"]], |
| 70 | + "environment": { |
| 71 | + key: os.environ.get(key, "") |
| 72 | + for key in ( |
| 73 | + "ImageOS", |
| 74 | + "ImageVersion", |
| 75 | + "RUSTFLAGS", |
| 76 | + "RUSTDOCFLAGS", |
| 77 | + "CARGO_BUILD_TARGET", |
| 78 | + "CC", |
| 79 | + "CXX", |
| 80 | + ) |
| 81 | + }, |
| 82 | + "workspace": str(ROOT), |
| 83 | + "roots": {key: str(value) for key, value in ROOTS.items()}, |
| 84 | + } |
| 85 | + |
| 86 | + |
| 87 | +def verify_sources(root=NATIVE): |
| 88 | + sys.path.insert(0, str(NATIVE / "scripts")) |
| 89 | + from source_integrity import verify_pristine_source, verify_rmcp_source |
| 90 | + |
| 91 | + verify_pristine_source(root) |
| 92 | + verify_rmcp_source(root) |
| 93 | + |
| 94 | + |
| 95 | +def load_downloader(): |
| 96 | + spec = importlib.util.spec_from_file_location( |
| 97 | + "native_snapshot", ROOT / ".github/scripts/download-native-snapshot.py" |
| 98 | + ) |
| 99 | + module = importlib.util.module_from_spec(spec) |
| 100 | + spec.loader.exec_module(module) |
| 101 | + return module.download_archive |
| 102 | + |
| 103 | + |
| 104 | +def unpack(archive): |
| 105 | + """Validate the entire archive before materializing any cached files.""" |
| 106 | + with tarfile.open(archive) as contents: |
| 107 | + members = contents.getmembers() |
| 108 | + names = set() |
| 109 | + if len(members) > 400_000 or sum(item.size for item in members) > 30 * 1024**3: |
| 110 | + raise ValueError("CodeQL Cargo cache exceeds extraction limits") |
| 111 | + for item in members: |
| 112 | + parts = PurePosixPath(item.name).parts |
| 113 | + if ( |
| 114 | + not parts |
| 115 | + or parts[0] not in ROOTS |
| 116 | + or item.name != "/".join(parts) |
| 117 | + or any(part in {".", ".."} for part in parts) |
| 118 | + or "\\" in item.name |
| 119 | + or item.name in names |
| 120 | + or not (item.isfile() or item.isdir() or item.issym()) |
| 121 | + ): |
| 122 | + raise ValueError("Unexpected CodeQL Cargo cache member") |
| 123 | + names.add(item.name) |
| 124 | + if item.issym(): |
| 125 | + destination = ROOTS[parts[0]].joinpath(*parts[1:]) |
| 126 | + if parts[0] != "upstream" or not ( |
| 127 | + destination.parent / item.linkname |
| 128 | + ).resolve().is_relative_to(ROOTS["upstream"].resolve()): |
| 129 | + raise ValueError("CodeQL Cargo cache link escapes native sources") |
| 130 | + with tempfile.TemporaryDirectory() as temporary: |
| 131 | + contents.extractall(temporary, filter="data") |
| 132 | + # Preserve the archived source mtimes alongside Cargo fingerprints. |
| 133 | + staged = Path(temporary) / "native" |
| 134 | + staged.mkdir() |
| 135 | + (staged / ".upstream").symlink_to( |
| 136 | + Path(temporary) / "upstream", target_is_directory=True |
| 137 | + ) |
| 138 | + for name in ("CODEX_REVISION", "Cargo.toml", "patches", "rmcp-patches"): |
| 139 | + source = NATIVE / name |
| 140 | + if source.is_dir(): |
| 141 | + shutil.copytree(source, staged / name) |
| 142 | + else: |
| 143 | + shutil.copy2(source, staged / name) |
| 144 | + verify_sources(staged) |
| 145 | + for key, destination in ROOTS.items(): |
| 146 | + source = Path(temporary) / key |
| 147 | + if not source.is_dir() or (key != "upstream" and destination.exists()): |
| 148 | + raise ValueError("CodeQL Cargo cache destination is not fresh") |
| 149 | + if ROOTS["upstream"].exists(): |
| 150 | + shutil.rmtree(ROOTS["upstream"]) |
| 151 | + for key, destination in ROOTS.items(): |
| 152 | + destination.parent.mkdir(parents=True, exist_ok=True) |
| 153 | + shutil.copytree( |
| 154 | + Path(temporary) / key, destination, symlinks=True, copy_function=shutil.copy2 |
| 155 | + ) |
| 156 | + verify_sources() |
| 157 | + |
| 158 | + |
| 159 | +def restore(): |
| 160 | + current = identity() |
| 161 | + workflow = api("actions/workflows/codeql.yml") |
| 162 | + runs = api( |
| 163 | + f"actions/workflows/codeql.yml/runs?branch={BRANCH}&event=push&status=success&per_page=5" |
| 164 | + )["workflow_runs"] |
| 165 | + for candidate in runs: |
| 166 | + if ( |
| 167 | + candidate["workflow_id"] != workflow["id"] |
| 168 | + or candidate["event"] != "push" |
| 169 | + or candidate["head_branch"] != BRANCH |
| 170 | + or candidate["head_repository"]["full_name"] != "openai/openai-agents-python" |
| 171 | + or candidate["path"] != WORKFLOW |
| 172 | + or candidate["conclusion"] != "success" |
| 173 | + or candidate["run_attempt"] != 1 |
| 174 | + ): |
| 175 | + continue |
| 176 | + run_id = str(candidate["id"]) |
| 177 | + run("git", "fetch", "--no-tags", "--depth=1", "origin", candidate["head_sha"]) |
| 178 | + if inputs(candidate["head_sha"]) != current["inputs"]: |
| 179 | + continue |
| 180 | + jobs = api(f"actions/runs/{run_id}/jobs?per_page=100") |
| 181 | + if jobs["total_count"] > 100: |
| 182 | + raise ValueError("Incomplete CodeQL job listing") |
| 183 | + rust = [ |
| 184 | + job |
| 185 | + for job in jobs["jobs"] |
| 186 | + if job["name"] == "CodeQL (rust)" and job["conclusion"] == "success" |
| 187 | + ] |
| 188 | + if len(rust) != 1 or not any( |
| 189 | + step["name"] == "Publish CodeQL Cargo cache" and step["conclusion"] == "success" |
| 190 | + for step in rust[0]["steps"] |
| 191 | + ): |
| 192 | + continue |
| 193 | + result = api(f"actions/runs/{run_id}/artifacts?per_page=100") |
| 194 | + if result["total_count"] > 100: |
| 195 | + raise ValueError("Incomplete CodeQL artifact listing") |
| 196 | + artifacts = [ |
| 197 | + item for item in result["artifacts"] if item["name"] == NAME and not item["expired"] |
| 198 | + ] |
| 199 | + if not artifacts: |
| 200 | + continue |
| 201 | + if len(artifacts) != 1: |
| 202 | + raise ValueError("Ambiguous CodeQL Cargo cache publication") |
| 203 | + artifact = artifacts[0] |
| 204 | + (publication,) = [ |
| 205 | + step for step in rust[0]["steps"] if step["name"] == "Publish CodeQL Cargo cache" |
| 206 | + ] |
| 207 | + times = [ |
| 208 | + datetime.fromisoformat(value.replace("Z", "+00:00")) |
| 209 | + for value in ( |
| 210 | + publication["started_at"], |
| 211 | + artifact["created_at"], |
| 212 | + publication["completed_at"], |
| 213 | + ) |
| 214 | + ] |
| 215 | + if not times[0] <= times[1] <= times[2]: |
| 216 | + raise ValueError("CodeQL Cargo cache was created outside the Rust publication step") |
| 217 | + with tempfile.TemporaryFile() as downloaded, tempfile.TemporaryDirectory() as temporary: |
| 218 | + load_downloader()(artifact["id"], artifact["digest"], run_id, NAME, downloaded) |
| 219 | + with ZipFile(downloaded) as bundle: |
| 220 | + if sorted(bundle.namelist()) != ["cargo.tar", "identity.json"]: |
| 221 | + raise ValueError("Unexpected CodeQL Cargo cache inventory") |
| 222 | + captured = json.loads(bundle.read("identity.json")) |
| 223 | + if captured != current: |
| 224 | + print("Selected CodeQL Cargo cache identity differs; extracting cold") |
| 225 | + return |
| 226 | + archive = Path(temporary) / "cargo.tar" |
| 227 | + with bundle.open("cargo.tar") as source, archive.open("wb") as target: |
| 228 | + shutil.copyfileobj(source, target) |
| 229 | + unpack(archive) |
| 230 | + print(f"Restored verified CodeQL Cargo cache from run {run_id}") |
| 231 | + return |
| 232 | + print("No equivalent canonical CodeQL Cargo cache; extracting cold") |
| 233 | + |
| 234 | + |
| 235 | +def pack(): |
| 236 | + verify_sources() |
| 237 | + # Package-scoped Cargo clean defaults to dev, so clean every observed layout. |
| 238 | + target = ROOTS["target"] |
| 239 | + fingerprints = sorted(target.glob("*/.fingerprint")) + sorted(target.glob("*/*/.fingerprint")) |
| 240 | + for directory in fingerprints: |
| 241 | + profile = directory.parent |
| 242 | + options = ["--profile", "dev" if profile.name == "debug" else profile.name] |
| 243 | + if profile.parent != target: |
| 244 | + options += ["--target", profile.parent.name] |
| 245 | + subprocess.run( |
| 246 | + [ |
| 247 | + "cargo", |
| 248 | + "clean", |
| 249 | + "--frozen", |
| 250 | + "--manifest-path", |
| 251 | + str(NATIVE / "Cargo.toml"), |
| 252 | + "--target-dir", |
| 253 | + str(target), |
| 254 | + "--package", |
| 255 | + "codex-agents-python", |
| 256 | + *options, |
| 257 | + ], |
| 258 | + check=True, |
| 259 | + ) |
| 260 | + destination = Path(os.environ["RUNNER_TEMP"]) / "codeql-cargo-publication" |
| 261 | + destination.mkdir() |
| 262 | + (destination / "identity.json").write_text(json.dumps(identity(), sort_keys=True)) |
| 263 | + with tarfile.open(destination / "cargo.tar", "w") as archive: |
| 264 | + for key, source in ROOTS.items(): |
| 265 | + source.mkdir(parents=True, exist_ok=True) |
| 266 | + for path in [source, *sorted(source.rglob("*"))]: |
| 267 | + if path.is_symlink() and key != "upstream": |
| 268 | + raise ValueError("Unexpected link in Cargo dependency cache") |
| 269 | + member = archive.gettarinfo( |
| 270 | + str(path), arcname=str(Path(key) / path.relative_to(source)) |
| 271 | + ) |
| 272 | + # Store hard-linked regular files independently. |
| 273 | + if member.islnk(): |
| 274 | + member.type, member.linkname, member.size = ( |
| 275 | + tarfile.REGTYPE, |
| 276 | + "", |
| 277 | + path.stat().st_size, |
| 278 | + ) |
| 279 | + if member.isfile(): |
| 280 | + with path.open("rb") as data: |
| 281 | + archive.addfile(member, data) |
| 282 | + elif member.isdir() or member.issym(): |
| 283 | + archive.addfile(member) |
| 284 | + else: |
| 285 | + raise ValueError("Unsupported Cargo cache file") |
| 286 | + |
| 287 | + |
| 288 | +if __name__ == "__main__": |
| 289 | + if sys.argv[1] == "restore": |
| 290 | + restore() |
| 291 | + elif sys.argv[1] == "pack": |
| 292 | + pack() |
| 293 | + else: |
| 294 | + raise SystemExit("Expected restore or pack") |
0 commit comments