Last active
May 3, 2026 22:57
-
-
Save maurycy/c03173b016ae4564a8600ea1cc3c7255 to your computer and use it in GitHub Desktop.
jsonl_batch_stats.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """Compute per-batch JSONL str_def + frame_def sizes for any Tachyon recording. | |
| Usage: | |
| python.exe jsonl_batch_stats.py <prof.bin> [batch_window_sec=5.0] [run_id=hex] | |
| Replays the binary and reports, for each time-bucketed batch, the bytes of | |
| str_def + frame_def needed to make that batch decodable in isolation | |
| (self-contained: every entry referenced by any sample in the batch is | |
| re-emitted, regardless of whether it was first seen earlier). This models a | |
| streaming format where each batch must stand alone, e.g. for a ring-buffer | |
| transport where older batches may be evicted before the consumer reads them. | |
| Format used: | |
| {"type":"str_def","v":1,"run_id":"<hex>","defs":[{"str_id":N,"value":"..."}]} | |
| {"type":"frame_def","v":1,"run_id":"<hex>","defs":[{"frame_id":N, | |
| "path_str_id":P,"func_str_id":F,"line":L,"end_line":L, | |
| "col":C,"end_col":C}]} | |
| """ | |
| import json | |
| import sys | |
| import uuid | |
| from profiling.sampling.binary_reader import BinaryReader | |
| from profiling.sampling.collector import extract_lineno | |
| from profiling.sampling.stack_collector import StackTraceCollector | |
| class BatchTracker(StackTraceCollector): | |
| """Track per-batch reference sets for strings and frames.""" | |
| def __init__(self, sample_interval_usec, batch_us): | |
| super().__init__(sample_interval_usec) | |
| self.batch_us = batch_us | |
| self.strings_seen = {} # str_value -> str_id (stable across run) | |
| self.frames_seen = {} # frame_key -> frame_id (stable across run) | |
| self.strings_per_batch = {} # batch_idx -> set of string values | |
| self.frames_per_batch = {} # batch_idx -> set of frame keys | |
| self.first_ts = None | |
| self.last_ts = None | |
| self._sample_idx = 0 | |
| self._interval = sample_interval_usec | |
| def collect(self, stack_frames, timestamps_us=None): | |
| if timestamps_us: | |
| ts = timestamps_us[0] | |
| weight = len(timestamps_us) | |
| else: | |
| ts = self._sample_idx * self._interval | |
| weight = 1 | |
| if self.first_ts is None: | |
| self.first_ts = ts | |
| self.last_ts = ts + (weight - 1) * self._interval | |
| batch_idx = (ts - self.first_ts) // self.batch_us | |
| strs = self.strings_per_batch.setdefault(batch_idx, set()) | |
| frms = self.frames_per_batch.setdefault(batch_idx, set()) | |
| for frames, thread_id in self._iter_stacks( | |
| stack_frames, skip_idle=self.skip_idle | |
| ): | |
| for frame in frames: | |
| filename, location, funcname, _ = frame | |
| if isinstance(location, (list, tuple)) and len(location) >= 4: | |
| line, end_line, col, end_col = ( | |
| location[0], | |
| location[1], | |
| location[2], | |
| location[3], | |
| ) | |
| else: | |
| line = extract_lineno(location) | |
| end_line, col, end_col = line, 0, 0 | |
| if filename not in self.strings_seen: | |
| self.strings_seen[filename] = len(self.strings_seen) + 1 | |
| if funcname not in self.strings_seen: | |
| self.strings_seen[funcname] = len(self.strings_seen) + 1 | |
| key = (filename, line, end_line, col, end_col, funcname) | |
| if key not in self.frames_seen: | |
| self.frames_seen[key] = len(self.frames_seen) + 1 | |
| strs.add(filename) | |
| strs.add(funcname) | |
| frms.add(key) | |
| self._sample_idx += weight | |
| def export(self, filename): | |
| pass | |
| def percentiles(values, ps=(0.5, 0.9, 0.95, 0.99)): | |
| if not values: | |
| return {p: 0 for p in ps} | |
| s = sorted(values) | |
| n = len(s) | |
| return {p: s[min(int(p * n), n - 1)] for p in ps} | |
| def fmt_bytes(n): | |
| for unit in ("B", "KB", "MB", "GB"): | |
| if n < 1024: | |
| return f"{n:.2f} {unit}" | |
| n /= 1024 | |
| return f"{n:.2f} TB" | |
| def encode_str_def(run_id, defs): | |
| rec = {"type": "str_def", "v": 1, "run_id": run_id, "defs": defs} | |
| return len((json.dumps(rec, ensure_ascii=False) + "\n").encode("utf-8")) | |
| def encode_frame_def(run_id, defs): | |
| rec = {"type": "frame_def", "v": 1, "run_id": run_id, "defs": defs} | |
| return len((json.dumps(rec, ensure_ascii=False) + "\n").encode("utf-8")) | |
| def main(): | |
| path = sys.argv[1] | |
| batch_sec = float(sys.argv[2]) if len(sys.argv) > 2 else 5.0 | |
| run_id = sys.argv[3] if len(sys.argv) > 3 else uuid.uuid4().hex | |
| batch_us = int(batch_sec * 1_000_000) | |
| with BinaryReader(path) as r: | |
| info = r.get_info() | |
| coll = BatchTracker(info["sample_interval_us"], batch_us) | |
| try: | |
| r.replay_samples(coll) | |
| except (ValueError, RuntimeError) as e: | |
| print(f"# partial replay: {e}", file=sys.stderr) | |
| if coll.first_ts is None: | |
| print("No samples replayed.", file=sys.stderr) | |
| sys.exit(1) | |
| duration_us = max(0, coll.last_ts - coll.first_ts) | |
| num_batches = max(1, (duration_us // batch_us) + 1) | |
| str_ids = coll.strings_seen | |
| frame_ids = coll.frames_seen | |
| str_per_batch = [ | |
| sorted(coll.strings_per_batch.get(b, set()), key=lambda s: str_ids[s]) | |
| for b in range(num_batches) | |
| ] | |
| frame_per_batch = [ | |
| sorted( | |
| coll.frames_per_batch.get(b, set()), key=lambda fr: frame_ids[fr] | |
| ) | |
| for b in range(num_batches) | |
| ] | |
| str_counts, str_bytes = [], [] | |
| frame_counts, frame_bytes = [], [] | |
| for b in range(num_batches): | |
| ss = str_per_batch[b] | |
| fs = frame_per_batch[b] | |
| str_counts.append(len(ss)) | |
| frame_counts.append(len(fs)) | |
| if ss: | |
| defs = [{"str_id": str_ids[s], "value": s} for s in ss] | |
| str_bytes.append(encode_str_def(run_id, defs)) | |
| else: | |
| str_bytes.append(0) | |
| if fs: | |
| defs = [] | |
| for fr in fs: | |
| fname, ln, eln, c, ec, fn = fr | |
| defs.append( | |
| { | |
| "frame_id": frame_ids[fr], | |
| "path_str_id": str_ids[fname], | |
| "func_str_id": str_ids[fn], | |
| "line": ln, | |
| "end_line": eln, | |
| "col": c, | |
| "end_col": ec, | |
| } | |
| ) | |
| frame_bytes.append(encode_frame_def(run_id, defs)) | |
| else: | |
| frame_bytes.append(0) | |
| combined_counts = [s + f for s, f in zip(str_counts, frame_counts)] | |
| combined_bytes = [s + f for s, f in zip(str_bytes, frame_bytes)] | |
| print(f"# JSONL batch stats: `{path}`") | |
| print() | |
| print("## Configuration") | |
| print() | |
| print("| Field | Value |") | |
| print("|---|---:|") | |
| print(f"| Wall-clock duration | {duration_us / 1_000_000:.2f} s |") | |
| print(f"| Batch window | {batch_sec} s |") | |
| print(f"| Number of batches | {num_batches} |") | |
| print(f"| Unique strings (whole run) | {len(coll.strings_seen):,} |") | |
| print(f"| Unique frames (whole run) | {len(coll.frames_seen):,} |") | |
| print(f"| run_id length | {len(run_id)} chars |") | |
| print() | |
| def report(name, counts, byte_vals): | |
| cp = percentiles(counts) | |
| bp = percentiles(byte_vals) | |
| non_zero = [v for v in byte_vals if v > 0] | |
| print(f"### `{name}` per batch") | |
| print() | |
| print(f"_Non-empty batches: {len(non_zero)} / {len(byte_vals)}_") | |
| print() | |
| print("| Stat | Entries | Bytes | |") | |
| print("|---|---:|---:|---|") | |
| rows = [ | |
| ("p50", cp[0.5], bp[0.5]), | |
| ("p90", cp[0.9], bp[0.9]), | |
| ("p95", cp[0.95], bp[0.95]), | |
| ("p99", cp[0.99], bp[0.99]), | |
| ( | |
| "max", | |
| max(counts) if counts else 0, | |
| max(byte_vals) if byte_vals else 0, | |
| ), | |
| ( | |
| "mean", | |
| (sum(counts) // len(counts)) if counts else 0, | |
| (sum(byte_vals) // len(byte_vals)) if byte_vals else 0, | |
| ), | |
| ("sum", sum(counts), sum(byte_vals)), | |
| ] | |
| for stat, c, b in rows: | |
| print(f"| **{stat}** | {c:,} | {b:,} | {fmt_bytes(b)} |") | |
| print() | |
| print("## Self-contained dictionaries per batch") | |
| print() | |
| report("str_def", str_counts, str_bytes) | |
| report("frame_def", frame_counts, frame_bytes) | |
| report("str_def + frame_def combined", combined_counts, combined_bytes) | |
| SAMPLE_BYTES_FULL = 200 | |
| SAMPLE_BYTES_ABBREV = 150 | |
| SAMPLING_RATES_HZ = (1_000, 10_000, 100_000, 1_000_000) | |
| p99 = percentiles(combined_bytes)[0.99] | |
| print( | |
| "## Dict overhead vs. sample stream (constant per-sample assumptions)" | |
| ) | |
| print() | |
| print( | |
| f"_Per-sample message size: full = {SAMPLE_BYTES_FULL} B, " | |
| f"abbreviated = {SAMPLE_BYTES_ABBREV} B (ballpark, ~20-frame stack)._" | |
| ) | |
| print() | |
| print( | |
| "_Overhead = dict_bytes / (dict_bytes + sample_bytes_in_batch). " | |
| "Computed for p99 dict bytes per batch and a single batch window._" | |
| ) | |
| print() | |
| print( | |
| "| Rate | Sample data / batch (full) | Sample data / batch (abbrev) " | |
| "| Dict % (v. full raw) | Dict % (v. abbreviated raw) |" | |
| ) | |
| print("|---|---:|---:|---:|---:|") | |
| for rate in SAMPLING_RATES_HZ: | |
| samples_per_batch = rate * batch_sec | |
| sample_bytes_full = samples_per_batch * SAMPLE_BYTES_FULL | |
| sample_bytes_abbrev = samples_per_batch * SAMPLE_BYTES_ABBREV | |
| pct_full = p99 / (p99 + sample_bytes_full) * 100 | |
| pct_abbrev = p99 / (p99 + sample_bytes_abbrev) * 100 | |
| rate_label = ( | |
| f"{rate // 1000} kHz" | |
| if rate < 1_000_000 | |
| else f"{rate // 1_000_000} MHz" | |
| ) | |
| print( | |
| f"| {rate_label} | {fmt_bytes(sample_bytes_full)} " | |
| f"| {fmt_bytes(sample_bytes_abbrev)} " | |
| f"| {pct_full:.2f}% | {pct_abbrev:.2f}% |" | |
| ) | |
| print() | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment