Created
May 27, 2026 00:37
-
-
Save abnobdoss/824698ea74c29c02ead8da6410b89b6e to your computer and use it in GitHub Desktop.
PyIceberg upsert match-filter recursion benchmark
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import argparse | |
| import json | |
| import subprocess | |
| import sys | |
| import textwrap | |
| import time | |
| ROW_COUNTS = [100, 1_000, 10_000, 100_000, 1_000_000] | |
| JOIN_COL_COUNTS = [1, 2, 3] | |
| def run_cell(kind: str, rows: int, join_cols: int, timeout: int) -> dict[str, object]: | |
| script = textwrap.dedent( | |
| f""" | |
| import json | |
| import time | |
| import pyarrow as pa | |
| from pyiceberg.expressions.visitors import bind | |
| from pyiceberg.io.pyarrow import expression_to_pyarrow | |
| from pyiceberg.schema import Schema | |
| from pyiceberg.table.upsert_util import create_file_match_filter, create_match_filter | |
| from pyiceberg.types import NestedField, LongType | |
| rows = {rows} | |
| join_cols = {join_cols} | |
| kind = {kind!r} | |
| arrays = {{}} | |
| fields = [] | |
| for idx in range(join_cols): | |
| name = f"k{{idx}}" | |
| arrays[name] = list(range(rows)) | |
| fields.append(NestedField(field_id=idx + 1, name=name, field_type=LongType(), required=False)) | |
| table = pa.table(arrays) | |
| cols = list(arrays) | |
| schema = Schema(*fields) | |
| fn = create_match_filter if kind == "exact" else create_file_match_filter | |
| construct_start = time.perf_counter() | |
| expr = fn(table, cols) | |
| construct_seconds = time.perf_counter() - construct_start | |
| bind_start = time.perf_counter() | |
| bound = bind(schema, expr, case_sensitive=True) | |
| bind_seconds = time.perf_counter() - bind_start | |
| arrow_start = time.perf_counter() | |
| arrow_expr = expression_to_pyarrow(bound) | |
| arrow_seconds = time.perf_counter() - arrow_start | |
| eval_start = time.perf_counter() | |
| table.filter(arrow_expr) | |
| eval_seconds = time.perf_counter() - eval_start | |
| print(json.dumps({{ | |
| "construct_seconds": construct_seconds, | |
| "bind_seconds": bind_seconds, | |
| "arrow_seconds": arrow_seconds, | |
| "eval_seconds": eval_seconds, | |
| "total_seconds": construct_seconds + bind_seconds + arrow_seconds + eval_seconds, | |
| "expr": type(expr).__name__, | |
| }})) | |
| """ | |
| ) | |
| started = time.perf_counter() | |
| try: | |
| proc = subprocess.run( | |
| [sys.executable, "-c", script], | |
| check=False, | |
| capture_output=True, | |
| text=True, | |
| timeout=timeout, | |
| ) | |
| except subprocess.TimeoutExpired: | |
| return {"kind": kind, "rows": rows, "join_cols": join_cols, "status": f">{timeout}s"} | |
| wall = time.perf_counter() - started | |
| if proc.returncode != 0: | |
| return { | |
| "kind": kind, | |
| "rows": rows, | |
| "join_cols": join_cols, | |
| "status": f"exit {proc.returncode}", | |
| "stderr": proc.stderr[-500:], | |
| "wall_seconds": wall, | |
| } | |
| payload = json.loads(proc.stdout) | |
| return { | |
| "kind": kind, | |
| "rows": rows, | |
| "join_cols": join_cols, | |
| "status": "ok", | |
| "wall_seconds": wall, | |
| **payload, | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--timeout", type=int, default=90) | |
| args = parser.parse_args() | |
| for kind in ["exact", "bounded"]: | |
| for rows in ROW_COUNTS: | |
| for join_cols in JOIN_COL_COUNTS: | |
| print(json.dumps(run_cell(kind, rows, join_cols, args.timeout)), flush=True) | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment