Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Select an option

  • Save abnobdoss/824698ea74c29c02ead8da6410b89b6e to your computer and use it in GitHub Desktop.

Select an option

Save abnobdoss/824698ea74c29c02ead8da6410b89b6e to your computer and use it in GitHub Desktop.
PyIceberg upsert match-filter recursion benchmark
import argparse
import json
import subprocess
import sys
import textwrap
import time
ROW_COUNTS = [100, 1_000, 10_000, 100_000, 1_000_000]
JOIN_COL_COUNTS = [1, 2, 3]
def run_cell(kind: str, rows: int, join_cols: int, timeout: int) -> dict[str, object]:
script = textwrap.dedent(
f"""
import json
import time
import pyarrow as pa
from pyiceberg.expressions.visitors import bind
from pyiceberg.io.pyarrow import expression_to_pyarrow
from pyiceberg.schema import Schema
from pyiceberg.table.upsert_util import create_file_match_filter, create_match_filter
from pyiceberg.types import NestedField, LongType
rows = {rows}
join_cols = {join_cols}
kind = {kind!r}
arrays = {{}}
fields = []
for idx in range(join_cols):
name = f"k{{idx}}"
arrays[name] = list(range(rows))
fields.append(NestedField(field_id=idx + 1, name=name, field_type=LongType(), required=False))
table = pa.table(arrays)
cols = list(arrays)
schema = Schema(*fields)
fn = create_match_filter if kind == "exact" else create_file_match_filter
construct_start = time.perf_counter()
expr = fn(table, cols)
construct_seconds = time.perf_counter() - construct_start
bind_start = time.perf_counter()
bound = bind(schema, expr, case_sensitive=True)
bind_seconds = time.perf_counter() - bind_start
arrow_start = time.perf_counter()
arrow_expr = expression_to_pyarrow(bound)
arrow_seconds = time.perf_counter() - arrow_start
eval_start = time.perf_counter()
table.filter(arrow_expr)
eval_seconds = time.perf_counter() - eval_start
print(json.dumps({{
"construct_seconds": construct_seconds,
"bind_seconds": bind_seconds,
"arrow_seconds": arrow_seconds,
"eval_seconds": eval_seconds,
"total_seconds": construct_seconds + bind_seconds + arrow_seconds + eval_seconds,
"expr": type(expr).__name__,
}}))
"""
)
started = time.perf_counter()
try:
proc = subprocess.run(
[sys.executable, "-c", script],
check=False,
capture_output=True,
text=True,
timeout=timeout,
)
except subprocess.TimeoutExpired:
return {"kind": kind, "rows": rows, "join_cols": join_cols, "status": f">{timeout}s"}
wall = time.perf_counter() - started
if proc.returncode != 0:
return {
"kind": kind,
"rows": rows,
"join_cols": join_cols,
"status": f"exit {proc.returncode}",
"stderr": proc.stderr[-500:],
"wall_seconds": wall,
}
payload = json.loads(proc.stdout)
return {
"kind": kind,
"rows": rows,
"join_cols": join_cols,
"status": "ok",
"wall_seconds": wall,
**payload,
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--timeout", type=int, default=90)
args = parser.parse_args()
for kind in ["exact", "bounded"]:
for rows in ROW_COUNTS:
for join_cols in JOIN_COL_COUNTS:
print(json.dumps(run_cell(kind, rows, join_cols, args.timeout)), flush=True)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment