import lancedb
import pyarrow as pa

db = lancedb.connect("/tmp/test_merge_compact")

schema = pa.schema([
    pa.field("vector", pa.list_(pa.float32(), 4), nullable=True),
    pa.field("path", pa.utf8()),
    pa.field("status", pa.utf8()),
    pa.field("file_size", pa.int64()),
])

# 1. Create table with 500 rows
tbl = db.create_table("test", schema=schema)
records = [{"vector": None, "path": f"/img/{i}.jpg", "status": "pending", "file_size": 1000 + i} for i in range(500)]
tbl.add(pa.Table.from_pylist(records, schema=schema))

# 2. Create scalar index on the merge key
tbl.create_scalar_index("path")

# 3. merge_insert 128 rows — WORKS
updates = [{"path": f"/img/{i}.jpg", "vector": [0.1]*4, "status": "indexed"} for i in range(128)]
tbl.merge_insert("path").when_matched_update_all().execute(pa.Table.from_pylist(updates))
# Verify: 128 indexed ✅

# 4. compact
from datetime import timedelta
tbl.optimize(cleanup_older_than=timedelta(minutes=0))

# 5. merge_insert 128 more — SILENTLY FAILS
updates2 = [{"path": f"/img/{i}.jpg", "vector": [0.2]*4, "status": "indexed"} for i in range(128, 256)]
tbl.merge_insert("path").when_matched_update_all().execute(pa.Table.from_pylist(updates2))

# Check: only 128 indexed, not 256
import pyarrow.compute as pc
arrow = tbl.to_arrow().select(["status"])
n_indexed = pc.sum(pc.equal(arrow.column("status"), "indexed")).as_py()
print(f"Indexed: {n_indexed}")  # Expected: 256, Actual: 128