Skip to content

Instantly share code, notes, and snippets.

@ariel-miculas
Created April 20, 2026 07:15
Show Gist options
  • Select an option

  • Save ariel-miculas/a7c7297e2d9e81c6e805ae9b5908b2c3 to your computer and use it in GitHub Desktop.

Select an option

Save ariel-miculas/a7c7297e2d9e81c6e805ae9b5908b2c3 to your computer and use it in GitHub Desktop.
#!/usr/bin/env python3
"""Truncate an NDJSON file to a target size (default 25 GiB), ending on a complete line."""
import sys
path = sys.argv[1] if len(sys.argv) > 1 else "/mnt/hits.json"
target_gib = float(sys.argv[2]) if len(sys.argv) > 2 else 25
target = int(target_gib * 1024 * 1024 * 1024)
output = sys.argv[3] if len(sys.argv) > 3 else "/mnt/hits_25g.json"
with open(path, "rb") as src, open(output, "wb") as dst:
remaining = target
while remaining > 0:
chunk = src.read(min(64 * 1024 * 1024, remaining))
if not chunk:
break
dst.write(chunk)
remaining -= len(chunk)
# Read forward to find the next newline, completing the last line
tail = src.read(10000)
nl = tail.find(b"\n")
if nl == -1:
print("ERROR: no newline found in 10000 bytes after target", file=sys.stderr)
sys.exit(1)
dst.write(tail[: nl + 1])
import os
size = os.path.getsize(output)
print(f"Written {size} bytes ({size / (1024**3):.2f} GiB) to {output}")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment