Skip to content

Instantly share code, notes, and snippets.

@khmyznikov
Created September 3, 2026 00:01
Show Gist options
  • Select an option

  • Save khmyznikov/92a44b09170e9b70fd1921e51284437b to your computer and use it in GitHub Desktop.

Select an option

Save khmyznikov/92a44b09170e9b70fd1921e51284437b to your computer and use it in GitHub Desktop.
Supporting benchmark and reproducer for pypa/pip#14286
"""Compare wheel bytecode compilation across two local pip source trees.
Run this on Windows with real-time antivirus protection in its normal state.
The generated wheel, virtual environment, and installed files are temporary.
Results include exact installed-file comparisons as well as timings.
"""
from __future__ import annotations
import argparse
import base64
import csv
import hashlib
import io
import json
import os
import shutil
import statistics
import subprocess
import sys
import tempfile
import time
import zipfile
from pathlib import Path
from typing import Any
def record_digest(data: bytes) -> str:
digest = hashlib.sha256(data).digest()
return "sha256=" + base64.urlsafe_b64encode(digest).decode().rstrip("=")
def build_wheel(path: Path, source_count: int) -> None:
files: dict[str, bytes] = {
"benchpkg-1.0.dist-info/METADATA": (
b"Metadata-Version: 2.1\nName: benchpkg\nVersion: 1.0\n"
),
"benchpkg-1.0.dist-info/WHEEL": (
b"Wheel-Version: 1.0\n"
b"Generator: pip-review\n"
b"Root-Is-Purelib: true\n"
b"Tag: py3-none-any\n"
),
}
for index in range(source_count):
files[f"benchpkg/module_{index:04d}.py"] = (
f"VALUE = {index}\n"
f"def calculate(value):\n"
" total = value\n"
" for offset in range(32):\n"
f" total = (total * 33 + offset + {index}) % 1000003\n"
" return total\n"
).encode()
record_path = "benchpkg-1.0.dist-info/RECORD"
rows = [
(name, record_digest(data), str(len(data)))
for name, data in sorted(files.items())
]
rows.append((record_path, "", ""))
record = io.StringIO(newline="")
csv.writer(record, lineterminator="\n").writerows(rows)
files[record_path] = record.getvalue().encode()
timestamp = (2024, 1, 1, 0, 0, 0)
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for name, data in sorted(files.items()):
info = zipfile.ZipInfo(name, timestamp)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
archive.writestr(info, data)
def get_defender_status() -> dict[str, Any] | None:
command = (
"$s=Get-MpComputerStatus -ErrorAction SilentlyContinue;"
"if($s){[pscustomobject]@{"
"antivirusEnabled=$s.AntivirusEnabled;"
"realTimeProtectionEnabled=$s.RealTimeProtectionEnabled;"
"ioavProtectionEnabled=$s.IoavProtectionEnabled;"
"behaviorMonitorEnabled=$s.BehaviorMonitorEnabled"
"}|ConvertTo-Json -Compress}"
)
result = subprocess.run(
["powershell", "-NoProfile", "-Command", command],
text=True,
capture_output=True,
)
if result.returncode or not result.stdout.strip():
return None
return json.loads(result.stdout)
def benchmark(
baseline: Path,
candidate: Path,
source_counts: list[int],
runs: int,
) -> dict[str, Any]:
with tempfile.TemporaryDirectory(prefix="pip-bytecode-benchmark-") as temp:
root = Path(temp)
wheel = root / "benchpkg-1.0-py3-none-any.whl"
environment = root / "environment"
subprocess.run([sys.executable, "-m", "venv", environment], check=True)
python = environment / "Scripts" / "python.exe"
site_packages = environment / "Lib" / "site-packages"
package_dir = site_packages / "benchpkg"
dist_info_dir = site_packages / "benchpkg-1.0.dist-info"
def clear_install() -> None:
shutil.rmtree(package_dir, ignore_errors=True)
shutil.rmtree(dist_info_dir, ignore_errors=True)
def snapshot() -> dict[str, str]:
installed = {}
for install_root in (package_dir, dist_info_dir):
installed.update(
{
path.relative_to(site_packages)
.as_posix(): hashlib.sha256(path.read_bytes())
.hexdigest()
for path in sorted(install_root.rglob("*"))
if path.is_file()
}
)
return installed
def install(source: Path, no_compile: bool) -> tuple[float, dict[str, str]]:
clear_install()
env = os.environ.copy()
env.update(
{
"PYTHONPATH": os.fspath(source / "src"),
"SOURCE_DATE_EPOCH": "1704067200",
"PIP_DISABLE_PIP_VERSION_CHECK": "1",
"PIP_NO_COLOR": "1",
"PYTHONUTF8": "1",
}
)
command = [
os.fspath(python),
"-m",
"pip",
"install",
"--no-index",
"--no-deps",
"--no-cache-dir",
]
if no_compile:
command.append("--no-compile")
command.append(os.fspath(wheel))
started = time.perf_counter()
result = subprocess.run(command, env=env, text=True, capture_output=True)
elapsed = time.perf_counter() - started
if result.returncode:
raise RuntimeError(f"{result.stdout}\n{result.stderr}")
return elapsed, snapshot()
def measure(
source_count: int, no_compile: bool, repetitions: int
) -> dict[str, Any]:
build_wheel(wheel, source_count)
install(baseline, no_compile)
install(candidate, no_compile)
timings: dict[str, list[float]] = {
"baseline": [],
"candidate": [],
}
snapshots: dict[str, list[dict[str, str]]] = {
"baseline": [],
"candidate": [],
}
sources = {"baseline": baseline, "candidate": candidate}
order = ["baseline", "candidate", "candidate", "baseline"]
while len(timings["baseline"]) < repetitions:
for name in order:
if len(timings[name]) >= repetitions:
continue
elapsed, installed = install(sources[name], no_compile)
timings[name].append(elapsed)
snapshots[name].append(installed)
reference = snapshots["baseline"][0]
artifacts_equal = all(
installed == reference
for condition in snapshots.values()
for installed in condition
)
baseline_median = statistics.median(timings["baseline"])
candidate_median = statistics.median(timings["candidate"])
return {
"sourceCount": source_count,
"noCompile": no_compile,
"baselineSeconds": timings["baseline"],
"candidateSeconds": timings["candidate"],
"baselineMedian": baseline_median,
"candidateMedian": candidate_median,
"savedSeconds": baseline_median - candidate_median,
"speedup": baseline_median / candidate_median,
"artifactsEqual": artifacts_equal,
"installedFileCount": len(reference),
}
compiled = [measure(count, False, runs) for count in source_counts]
no_compile = measure(max(source_counts), True, max(3, runs // 2))
return {
"python": sys.version,
"cpuCount": os.cpu_count(),
"processCpuCount": getattr(os, "process_cpu_count", os.cpu_count)(),
"defender": get_defender_status(),
"compiled": compiled,
"noCompileControl": no_compile,
}
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--baseline",
type=Path,
required=True,
help="Baseline pip source tree.",
)
parser.add_argument(
"--candidate",
type=Path,
required=True,
help="Candidate pip source tree.",
)
parser.add_argument(
"--counts",
default="7,8,32,128,256,1200",
help="Comma-separated Python source-file counts.",
)
parser.add_argument(
"--runs",
type=int,
default=6,
help="Measured runs per source tree and file count.",
)
args = parser.parse_args()
if sys.platform != "win32":
parser.error("this benchmark targets Windows bytecode compilation")
for source_tree in (args.baseline, args.candidate):
if not (source_tree / "src" / "pip").is_dir():
parser.error(f"not a pip source tree: {source_tree}")
counts = [int(value) for value in args.counts.split(",")]
print(
json.dumps(
benchmark(args.baseline, args.candidate, counts, args.runs),
separators=(",", ":"),
)
)

Standalone threaded bytecode determinism reproducer

repro_threaded_pyc_determinism.py isolates byte differences observed when a free-threaded CPython build compiles identical source files serially and with a thread pool. It uses only the Python standard library and does not import, execute, or otherwise involve pip.

The script:

  • Generates 64 copies of a minimized source fixture by default.
  • Runs serial and threaded compilation in fresh child interpreters.
  • Uses fixed logical filenames, PYTHONHASHSEED=0, and SOURCE_DATE_EPOCH=1704067200.
  • Produces either checked-hash .pyc files with py_compile or raw marshal.dumps(compile(...)) payloads.
  • Compares the bytes and recursively compares documented code-object fields after unmarshalling.
  • Preserves result.json and the first differing serial/threaded pair.

Reproduction

Use a new or empty output directory for each invocation. The script refuses to remove a non-empty output directory.

Free-threaded build with the GIL disabled:

python3.14t tools/repro_threaded_pyc_determinism.py --gil off --backend py_compile --workers 4 --output repro-ft-off-pyc --require-difference

The same build with its GIL re-enabled:

python3.14t tools/repro_threaded_pyc_determinism.py --gil on --backend py_compile --workers 4 --output repro-ft-on-pyc --require-difference

To bypass .pyc header generation and py_compile, use the raw marshal backend:

python3.14t tools/repro_threaded_pyc_determinism.py --gil off --backend marshal --workers 4 --output repro-ft-off-marshal --require-difference
python3.14t tools/repro_threaded_pyc_determinism.py --gil on --backend marshal --workers 4 --output repro-ft-on-marshal --require-difference

Run a standard-build control without --require-difference:

python3.14 tools/repro_threaded_pyc_determinism.py --backend py_compile --workers 4 --output repro-standard-pyc
python3.14 tools/repro_threaded_pyc_determinism.py --backend marshal --workers 4 --output repro-standard-marshal

Set --workers to the number of processors available to the process. Other useful options are --files, --attempts, and --fixture. A run with --require-difference exits with status 1 if no difference is found.

Result format

result.json records the Python and platform versions, available CPU count, free-threaded build marker, requested and observed GIL state, workload, and each attempt's differing-file count. For a difference it also records hashes, sizes, changed offsets, header equality, and semanticDifferences.

An empty semanticDifferences list means the recursive comparison found equal values for the inspected code-object fields, including bytecode, constants, names, filenames, line tables, exception tables, and nested code objects. It does not prove that the object identity or alias graph is identical.

Observed Windows results

Environment: Windows 11 prerelease VM, eight virtual CPUs, CPython 3.15.0rc1 free-threaded build, and the corresponding standard CPython 3.15 build. Each case used 64 files and eight workers.

Build Requested GIL Backend Outcome
Free-threaded Off py_compile Reproduced on attempt 1; 49 differing bytes in a 3,447-byte .pyc; identical 16-byte headers
Free-threaded On py_compile Reproduced on attempt 2; 49 differing bytes in a 3,447-byte .pyc; identical 16-byte headers
Free-threaded Off marshal Reproduced on attempt 2; 49 differing bytes in a 3,431-byte payload
Free-threaded On marshal Reproduced on attempt 1; 49 differing bytes in a 3,431-byte payload
Standard Default/on py_compile No difference in 10 attempts
Standard Default/on marshal No difference in 10 attempts

The changed raw-marshal offsets were exactly 16 bytes earlier than the corresponding .pyc offsets, confirming that the .pyc header was not the source of the difference. All inspected decoded code-object field values were equal.

Observed WSL results

Environment: Ubuntu 26.04 LTS on WSL2, kernel 6.18.33.2-microsoft-standard-WSL2, native ext4 storage, and four available CPUs. Each case used 64 files and four workers.

Build Requested GIL Backend Outcome
CPython 3.14.7 free-threaded Off py_compile Reproduced on attempt 1 in 2 files; 50 differing bytes per 3,343-byte .pyc; identical 16-byte headers
CPython 3.14.7 free-threaded On py_compile Reproduced on attempt 1 in 2 files; 50 differing bytes per 3,343-byte .pyc; identical 16-byte headers
CPython 3.14.7 free-threaded Off marshal Reproduced on attempt 1 in 2 files; 50 differing bytes per 3,327-byte payload
CPython 3.14.7 free-threaded On marshal Reproduced on attempt 1 in 2 files; 50 differing bytes per 3,327-byte payload
CPython 3.14.4 standard Default/on py_compile No difference in 10 attempts
CPython 3.14.4 standard Default/on marshal No difference in 10 attempts

All inspected decoded code-object field values were equal in the WSL differences as well.

Interpretation

The raw marshal backend reproduces without pip, py_compile, or .pyc headers, so the behavior is within concurrent compilation and/or marshal's representation of the resulting code-object graph. Re-enabling the GIL on a free-threaded build does not prevent it on either Windows or Linux.

The precise CPython cause and runtime significance are not established. However, byte-different outputs are sufficient to violate deterministic build, artifact hashing, caching, and reproducible-CI expectations. Until the cause is understood, build-time detection with sysconfig.get_config_var("Py_GIL_DISABLED") is safer than checking only the runtime GIL state before enabling threaded bytecode compilation.

"""Reproduce free-threaded CPython serial-versus-threaded marshal differences.
Examples:
python3.15t tools/repro_threaded_pyc_determinism.py --gil off \
--output repro-gil-off --require-difference
python3.15t tools/repro_threaded_pyc_determinism.py --gil on \
--backend marshal --output repro-gil-on --require-difference
The script uses only the Python standard library. It creates identical source
files, compiles them in fresh serial and threaded child processes, compares the
resulting bytes, and preserves the first differing pair.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import marshal
import os
import platform
import py_compile
import shutil
import subprocess
import sys
import sysconfig
import tempfile
import types
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any
DEFAULT_SOURCE = '''from __future__ import annotations
def validate_values(values: Mapping[str, Any]) -> Mapping[str, Any]:
if not values:
raise ValidationError("At least one value must be provided")
if not all(isinstance(item, str) for item in values.values()):
raise ValidationError("Values must be strings")
return values
class ValidationError(Exception):
"""Raised when input data is invalid."""
context: str | None = None
message: str
def __init__(
self,
cause: str | Exception,
*,
context: str | None = None,
) -> None:
if isinstance(cause, ValidationError):
if cause.context:
self.context = (
f"{context}.{cause.context}" if context else cause.context
)
else:
self.context = context
self.message = cause.message
else:
self.context = context
self.message = str(cause)
def __str__(self) -> str:
if self.context:
return f"{self.message} in {self.context!r}"
return self.message
class RequiredKeyError(ValidationError):
def __init__(self, key: str) -> None:
super().__init__("Missing required value", context=key)
'''
def positive_int(value: str) -> int:
parsed = int(value)
if parsed < 1:
raise argparse.ArgumentTypeError("must be at least 1")
return parsed
def compile_one(
source: Path,
destination: Path,
logical_name: str,
backend: str,
) -> None:
if backend == "py_compile":
py_compile.compile(
str(source),
cfile=str(destination),
dfile=logical_name,
doraise=True,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
)
else:
code = compile(
source.read_bytes(),
logical_name,
"exec",
dont_inherit=True,
)
destination.write_bytes(marshal.dumps(code))
def compile_sources(
sources: list[Path],
destination: Path,
*,
threaded: bool,
workers: int,
backend: str,
) -> None:
destination.mkdir(parents=True, exist_ok=True)
def compile_index(index: int) -> None:
compile_one(
sources[index],
destination / f"module_{index:03}.pyc",
f"/repro/module_{index:03}.py",
backend,
)
if threaded:
with ThreadPoolExecutor(max_workers=workers) as executor:
list(executor.map(compile_index, range(len(sources))))
else:
for index in range(len(sources)):
compile_index(index)
def compare_values(left: Any, right: Any, path: str, differences: list[str]) -> None:
if isinstance(left, types.CodeType) and isinstance(right, types.CodeType):
compare_code(left, right, path, differences)
elif (
isinstance(left, tuple) and isinstance(right, tuple) and len(left) == len(right)
):
for index, (left_item, right_item) in enumerate(zip(left, right)):
compare_values(left_item, right_item, f"{path}[{index}]", differences)
elif left != right:
differences.append(path)
def compare_code(
left: types.CodeType,
right: types.CodeType,
path: str,
differences: list[str],
) -> None:
fields = (
"co_argcount",
"co_posonlyargcount",
"co_kwonlyargcount",
"co_nlocals",
"co_stacksize",
"co_flags",
"co_code",
"co_consts",
"co_names",
"co_varnames",
"co_filename",
"co_name",
"co_qualname",
"co_firstlineno",
"co_linetable",
"co_exceptiontable",
"co_freevars",
"co_cellvars",
)
for field in fields:
compare_values(
getattr(left, field),
getattr(right, field),
f"{path}.{field}",
differences,
)
def compare_outputs(
serial_dir: Path,
threaded_dir: Path,
header_bytes: int,
) -> list[dict[str, Any]]:
differences = []
for serial_path in sorted(serial_dir.glob("*.pyc")):
threaded_path = threaded_dir / serial_path.name
serial = serial_path.read_bytes()
threaded = threaded_path.read_bytes()
if serial == threaded:
continue
semantic_differences: list[str] = []
compare_code(
marshal.loads(serial[header_bytes:]),
marshal.loads(threaded[header_bytes:]),
"code",
semantic_differences,
)
changed_offsets = [
index
for index, (left, right) in enumerate(zip(serial, threaded))
if left != right
]
differences.append(
{
"file": serial_path.name,
"serialSha256": hashlib.sha256(serial).hexdigest(),
"threadedSha256": hashlib.sha256(threaded).hexdigest(),
"serialBytes": len(serial),
"threadedBytes": len(threaded),
"headerBytes": header_bytes,
"headersEqual": (
serial[:header_bytes] == threaded[:header_bytes]
if header_bytes
else None
),
"differentByteCount": len(changed_offsets)
+ abs(len(serial) - len(threaded)),
"firstDifferentOffsets": changed_offsets[:50],
"semanticDifferences": semantic_differences[:100],
}
)
return differences
def child_command(
script: Path,
mode: str,
root: Path,
workers: int,
gil: str,
backend: str,
) -> list[str]:
command = [sys.executable]
if gil != "default":
command.extend(["-X", f"gil={1 if gil == 'on' else 0}"])
command.extend(
[
os.fspath(script),
"--child",
mode,
"--root",
os.fspath(root),
"--workers",
str(workers),
"--backend",
backend,
]
)
return command
def run_child(args: argparse.Namespace) -> None:
root = args.root
sources = sorted((root / "sources").glob("*.py"))
compile_sources(
sources,
root / args.child,
threaded=args.child == "threaded",
workers=args.workers,
backend=args.backend,
)
print(
json.dumps(
{
"mode": args.child,
"gilEnabled": getattr(sys, "_is_gil_enabled", lambda: True)(),
"files": len(sources),
},
separators=(",", ":"),
)
)
def run_parent(args: argparse.Namespace) -> int:
script = Path(__file__).resolve()
temporary = None
if args.output is None:
temporary = tempfile.TemporaryDirectory(prefix="threaded-pyc-repro-")
output = Path(temporary.name)
else:
output = args.output.resolve()
if output.exists() and any(output.iterdir()):
raise SystemExit(f"output directory is not empty: {output}")
output.mkdir(parents=True, exist_ok=True)
source_text = (
args.fixture.read_text(encoding="utf-8")
if args.fixture is not None
else DEFAULT_SOURCE
)
sources = output / "sources"
sources.mkdir()
for index in range(args.files):
(sources / f"module_{index:03}.py").write_text(source_text, encoding="utf-8")
environment = os.environ.copy()
environment.update(
{
"PYTHONHASHSEED": "0",
"SOURCE_DATE_EPOCH": "1704067200",
"PYTHONUTF8": "1",
}
)
attempts = []
first_difference = None
for attempt in range(1, args.attempts + 1):
serial_dir = output / "serial"
threaded_dir = output / "threaded"
shutil.rmtree(serial_dir, ignore_errors=True)
shutil.rmtree(threaded_dir, ignore_errors=True)
serial = subprocess.run(
child_command(
script,
"serial",
output,
args.workers,
args.gil,
args.backend,
),
env=environment,
text=True,
capture_output=True,
check=True,
)
threaded = subprocess.run(
child_command(
script,
"threaded",
output,
args.workers,
args.gil,
args.backend,
),
env=environment,
text=True,
capture_output=True,
check=True,
)
differences = compare_outputs(
serial_dir,
threaded_dir,
16 if args.backend == "py_compile" else 0,
)
attempts.append(
{
"attempt": attempt,
"serialChild": json.loads(serial.stdout),
"threadedChild": json.loads(threaded.stdout),
"differentFiles": len(differences),
}
)
if differences:
first_difference = differences
pair = output / "first-difference"
pair.mkdir()
name = differences[0]["file"]
shutil.copy2(serial_dir / name, pair / f"serial-{name}")
shutil.copy2(threaded_dir / name, pair / f"threaded-{name}")
break
result = {
"python": sys.version,
"platform": platform.platform(),
"availableCpuCount": getattr(os, "process_cpu_count", os.cpu_count)(),
"freeThreadedBuild": bool(sysconfig.get_config_var("Py_GIL_DISABLED")),
"requestedGil": args.gil,
"backend": args.backend,
"workers": args.workers,
"files": args.files,
"fixture": os.fspath(args.fixture) if args.fixture is not None else None,
"reproduced": first_difference is not None,
"attempts": attempts,
"differences": first_difference or [],
"output": os.fspath(output),
}
(output / "result.json").write_text(
json.dumps(result, indent=2),
encoding="utf-8",
)
print(json.dumps(result, indent=2))
if temporary is not None:
temporary.cleanup()
return 0 if result["reproduced"] or not args.require_difference else 1
parser = argparse.ArgumentParser(
description="Reproduce threaded py_compile byte differences without pip."
)
parser.add_argument("--child", choices=["serial", "threaded"], help=argparse.SUPPRESS)
parser.add_argument("--root", type=Path, help=argparse.SUPPRESS)
parser.add_argument("--workers", type=positive_int, default=8)
parser.add_argument("--files", type=positive_int, default=64)
parser.add_argument("--attempts", type=positive_int, default=10)
parser.add_argument("--gil", choices=["default", "on", "off"], default="default")
parser.add_argument(
"--backend",
choices=["py_compile", "marshal"],
default="py_compile",
)
parser.add_argument("--fixture", type=Path)
parser.add_argument("--output", type=Path)
parser.add_argument("--require-difference", action="store_true")
arguments = parser.parse_args()
if arguments.child:
if arguments.root is None:
parser.error("--root is required in child mode")
run_child(arguments)
else:
raise SystemExit(run_parent(arguments))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment