|
#!/usr/bin/env python3 |
|
""" |
|
A/B benchmark for the numpy.random Limited API (abi3) port. |
|
|
|
Compares an abi3 build against the normal build. Noise is attacked on five |
|
fronts, because a naive timeit run on this workload is good for ~50% error: |
|
|
|
1. Every measurement runs in its own subprocess, so per-process luck (ASLR, |
|
heap layout, hash seed) becomes a sampled variable instead of a constant |
|
baked into one build's number. |
|
2. The two builds are interleaved and the order is flipped every trial |
|
(ABBA), so CPU thermal drift and frequency scaling cancel out instead of |
|
landing entirely on whichever build ran second. |
|
3. Inside a process each case is auto-calibrated to run for ~50ms, warmed up, |
|
then min-of-N -- min is the right estimator here because noise is strictly |
|
additive (nothing makes the CPU faster than an uncontended run). |
|
4. The child pins itself to one core and disables the GC while timing. |
|
5. The ratio is reported with a bootstrap 95% CI, so you can tell a real 15% |
|
regression from a run of bad luck. If the CI spans 1.0, you have not |
|
measured anything. |
|
|
|
Usage |
|
----- |
|
python bench_limited_api.py --abi3 <site-packages-of-abi3-build> |
|
|
|
The default build is whatever `import numpy` already gives you (i.e. the |
|
editable install). The abi3 path is the site-packages directory of a |
|
`meson install --destdir` tree built with `-Dpython.allow_limited_api=true`. |
|
|
|
Run `--help` for knobs. Defaults (15 trials) take roughly two minutes. |
|
""" |
|
|
|
from __future__ import annotations |
|
|
|
import argparse |
|
import json |
|
import os |
|
import random |
|
import statistics |
|
import subprocess |
|
import sys |
|
import tempfile |
|
import textwrap |
|
from pathlib import Path |
|
|
|
# -------------------------------------------------------------------------- |
|
# Workloads. Split into "per-call" cases, where Python/Cython call overhead |
|
# dominates and the Limited API is expected to cost something, and "bulk" |
|
# cases, where the inner C loop dominates and it is expected to cost nothing. |
|
# Keeping both in the table is the point: a regression that shows up in one |
|
# and not the other tells you *where* the cost is. |
|
# -------------------------------------------------------------------------- |
|
CASES = { |
|
# name: (setup, statement, kind) |
|
"random() scalar": ("", "g.random()", "per-call"), |
|
"standard_normal()": ("", "g.standard_normal()", "per-call"), |
|
"integers(10)": ("", "g.integers(10)", "per-call"), |
|
"random(10)": ("", "g.random(10)", "per-call"), |
|
"default_rng()": ("", "nr.default_rng(2)", "per-call"), |
|
"SeedSequence.spawn": ("s = nr.SeedSequence(1)", "s.spawn(1)", "per-call"), |
|
"choice(1000,100)": ("", "g.choice(1000, 100)", "per-call"), |
|
"random(100k)": ("", "g.random(100_000)", "bulk"), |
|
"standard_normal(100k)": ("", "g.standard_normal(100_000)", "bulk"), |
|
"integers(0,100,100k)": ("", "g.integers(0, 100, 100_000)", "bulk"), |
|
"standard_gamma(100k)": ("", "g.standard_gamma(2.0, 100_000)", "bulk"), |
|
"shuffle(10k)": ("a = np.arange(10_000)", "g.shuffle(a)", "bulk"), |
|
"mvhg count": ("", "g.multivariate_hypergeometric([300,200,100], 250," |
|
" size=200, method='count')", "bulk"), |
|
"RandomState.binomial": ("rs = nr.RandomState(1)", |
|
"rs.binomial(100, 0.4, 10_000)", "bulk"), |
|
} |
|
|
|
# Body of the child process. Kept as a string so the child never imports this |
|
# module (and so never drags the parent's numpy into the measured process). |
|
CHILD = r''' |
|
import gc, json, os, sys, time, timeit |
|
|
|
target_ns = float(os.environ["BENCH_TARGET_NS"]) |
|
inner = int(os.environ["BENCH_INNER"]) |
|
core = os.environ.get("BENCH_CORE") |
|
|
|
if core and hasattr(os, "sched_setaffinity"): |
|
try: |
|
os.sched_setaffinity(0, {int(core)}) |
|
except OSError: |
|
pass |
|
|
|
import numpy as np |
|
import numpy.random as nr |
|
|
|
# Report which build actually got imported. Guessing here is how you end up |
|
# benchmarking the same build against itself for twenty minutes. |
|
tag = "abi3" if nr._generator.__file__.endswith("abi3.so") else "default" |
|
|
|
cases = json.loads(os.environ["BENCH_CASES"]) |
|
out = {} |
|
gc.disable() |
|
try: |
|
for name, (setup, stmt) in cases.items(): |
|
ns = {"np": np, "nr": nr, "g": nr.default_rng(1)} |
|
t = timeit.Timer(stmt, setup=setup or "pass", globals=ns) |
|
# Calibrate: grow `number` until one batch clears the target duration, |
|
# so short cases are not measured against the clock's resolution. |
|
number = 1 |
|
while True: |
|
dt = t.timeit(number) |
|
if dt >= target_ns / 1e9: |
|
break |
|
number = max(number * 2, int(number * (target_ns / 1e9) / max(dt, 1e-9)) + 1) |
|
if number > 50_000_000: |
|
break |
|
t.timeit(number) # warmup at the final `number` |
|
best = min(t.timeit(number) for _ in range(inner)) / number |
|
out[name] = best |
|
finally: |
|
gc.enable() |
|
|
|
json.dump({"tag": tag, "times": out}, sys.stdout) |
|
''' |
|
|
|
|
|
def make_hook(tmpdir: Path) -> Path: |
|
""" |
|
A sitecustomize that removes the editable-install meta path finder. |
|
|
|
Without this, PYTHONPATH silently loses to numpy's `numpy-editable.pth` |
|
and both halves of the A/B measure the same build. |
|
""" |
|
hook = tmpdir / "hook" |
|
hook.mkdir(exist_ok=True) |
|
(hook / "sitecustomize.py").write_text(textwrap.dedent(""" |
|
import sys |
|
sys.meta_path = [ |
|
f for f in sys.meta_path |
|
if 'editable' not in type(f).__module__ |
|
] |
|
""")) |
|
return hook |
|
|
|
|
|
def run_child(pythonpath, cases, target_ns, inner, core, hook): |
|
env = dict(os.environ) |
|
if pythonpath: |
|
# The hook is only wanted when we are overriding with an explicit |
|
# tree: it disables the editable-install finder, which for the |
|
# un-overridden build is the very thing that provides numpy. |
|
env["PYTHONPATH"] = os.pathsep.join([str(hook), str(pythonpath)]) |
|
else: |
|
env.pop("PYTHONPATH", None) |
|
env["PYTHONHASHSEED"] = "0" |
|
env["BENCH_CASES"] = json.dumps({k: (v[0], v[1]) for k, v in cases.items()}) |
|
env["BENCH_TARGET_NS"] = str(target_ns) |
|
env["BENCH_INNER"] = str(inner) |
|
if core is not None: |
|
env["BENCH_CORE"] = str(core) |
|
# cwd=/tmp so a stray ./numpy source dir cannot shadow the install |
|
r = subprocess.run([sys.executable, "-c", CHILD], env=env, cwd="/tmp", |
|
capture_output=True, text=True) |
|
if r.returncode != 0: |
|
sys.exit(f"child failed:\n{r.stderr}") |
|
return json.loads(r.stdout) |
|
|
|
|
|
def bootstrap_ci(a_samples, b_samples, n=10000, seed=0): |
|
"""95% CI for median(b)/median(a) by resampling both sides.""" |
|
rng = random.Random(seed) |
|
ratios = [] |
|
for _ in range(n): |
|
ra = statistics.median(rng.choices(a_samples, k=len(a_samples))) |
|
rb = statistics.median(rng.choices(b_samples, k=len(b_samples))) |
|
ratios.append(rb / ra) |
|
ratios.sort() |
|
return ratios[int(0.025 * n)], ratios[int(0.975 * n)] |
|
|
|
|
|
def main(): |
|
p = argparse.ArgumentParser( |
|
description="A/B benchmark the numpy.random Limited API port.") |
|
p.add_argument("--abi3", required=True, type=Path, |
|
help="site-packages dir of the abi3 (limited API) build") |
|
p.add_argument("--default", type=Path, default=None, |
|
help="site-packages dir of the reference build " |
|
"(default: whatever `import numpy` finds)") |
|
p.add_argument("--trials", type=int, default=15, |
|
help="processes per build (default: 15)") |
|
p.add_argument("--inner", type=int, default=7, |
|
help="min-of-N repeats inside each process (default: 7)") |
|
p.add_argument("--target-ms", type=float, default=50.0, |
|
help="calibrate each case to this batch duration " |
|
"(default: 50ms)") |
|
p.add_argument("--core", type=int, default=None, |
|
help="pin children to this CPU core, e.g. --core 3") |
|
p.add_argument("--only", choices=["per-call", "bulk"], default=None, |
|
help="restrict to one class of workload") |
|
p.add_argument("--json", type=Path, default=None, |
|
help="also write raw samples here") |
|
args = p.parse_args() |
|
|
|
cases = {k: v for k, v in CASES.items() |
|
if args.only is None or v[2] == args.only} |
|
|
|
with tempfile.TemporaryDirectory() as td: |
|
hook = make_hook(Path(td)) |
|
samples = {"default": {k: [] for k in cases}, |
|
"abi3": {k: [] for k in cases}} |
|
seen_tags = set() |
|
|
|
for trial in range(args.trials): |
|
# ABBA: flip the order every trial so drift cannot favour one side. |
|
order = [("default", args.default), ("abi3", args.abi3)] |
|
if trial % 2: |
|
order.reverse() |
|
for label, path in order: |
|
res = run_child(path, cases, args.target_ms * 1e6, |
|
args.inner, args.core, hook) |
|
seen_tags.add((label, res["tag"])) |
|
if res["tag"] != label: |
|
sys.exit( |
|
f"expected the '{label}' build but imported " |
|
f"'{res['tag']}'. Check --{label} path; the two sides " |
|
f"are not distinct builds.") |
|
for k, v in res["times"].items(): |
|
samples[label][k].append(v) |
|
if sys.stderr.isatty(): |
|
print(f"\rtrial {trial + 1}/{args.trials}", |
|
end="", file=sys.stderr, flush=True) |
|
if sys.stderr.isatty(): |
|
print("\r" + " " * 30 + "\r", end="", file=sys.stderr, flush=True) |
|
|
|
hdr = f"{'case':24s} {'kind':9s} {'default':>10s} {'abi3':>10s} {'ratio':>7s} 95% CI" |
|
print(hdr) |
|
print("-" * len(hdr)) |
|
for name in cases: |
|
a, b = samples["default"][name], samples["abi3"][name] |
|
ma, mb = statistics.median(a), statistics.median(b) |
|
lo, hi = bootstrap_ci(a, b) |
|
# A CI that straddles 1.0 means the difference is not resolved. |
|
flag = "" if lo <= 1.0 <= hi else (" <-- slower" if lo > 1.0 else " <-- faster") |
|
print(f"{name:24s} {cases[name][2]:9s} {ma * 1e6:8.2f}us {mb * 1e6:8.2f}us " |
|
f"{mb / ma:6.2f}x [{lo:.2f}, {hi:.2f}]{flag}") |
|
|
|
print(f"\n{args.trials} trials/build, min-of-{args.inner} per trial, " |
|
f"~{args.target_ms:.0f}ms batches, ABBA-interleaved.") |
|
print("Ratios whose CI includes 1.00 are not resolved by this run; " |
|
"raise --trials.") |
|
|
|
if args.json: |
|
args.json.write_text(json.dumps(samples, indent=1)) |
|
print(f"raw samples -> {args.json}") |
|
|
|
|
|
if __name__ == "__main__": |
|
main() |