Skip to content

Commit 3e49790

Browse files
committed
Fix SyntheticDataKit.chunk_data emitting chunks over max_tokens
The multi-chunk path built boundaries from np.linspace(..., n_chunks), but pairing boundaries[:-1] with boundaries[1:] turns N points into N-1 ranges, so it produced one fewer, oversized chunk: every chunk exceeded max_tokens and a document just over the threshold came back as a single unsplit chunk. Use n_chunks + 1 points so exactly n_chunks ranges are emitted, each within max_tokens. Also base n_chunks on the non-overlapped span: consecutive chunks overlap by overlap, so covering length needs ceil((length - overlap) / stride) chunks, not ceil(length / stride). The looser count over-counted by one just past a stride multiple (a 673-token doc became 3 chunks of ~267 instead of 2 of ~369), emitting an extra redundant chunk. Coverage and overlap are unchanged and every chunk still stays within max_tokens.
1 parent d105bd7 commit 3e49790

2 files changed

Lines changed: 49 additions & 2 deletions

File tree

‎tests/test_synthetic_chunk_data.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,50 @@ def test_chunk_data_uninitialized_error_names_real_class():
129129
os.unlink(path)
130130

131131

132+
def test_chunk_data_chunks_do_not_exceed_max_tokens():
133+
# Every chunk must fit within max_tokens (= max_seq_length - 2*max_generation_tokens - 128).
134+
# The multi-chunk path fed n_chunks points into linspace, but pairing boundaries[:-1] with
135+
# boundaries[1:] turns N points into N-1 ranges, so it emitted one fewer, oversized chunk:
136+
# chunks exceeded max_tokens and a document just over the threshold came back unsplit.
137+
kit = _make_kit(max_seq_length = 2048, max_generation_tokens = 760, overlap = 64)
138+
max_tokens = 2048 - 760 * 2 - 128 # 400
139+
140+
for n_words in (500, 2000):
141+
out, contents = _chunk("word " * n_words, kit = kit)
142+
assert (
143+
len(out) >= 2
144+
), f"a {n_words}-token doc (> max_tokens={max_tokens}) must be split, got {len(out)}"
145+
for content in contents:
146+
n_tokens = len(content.split())
147+
assert (
148+
n_tokens <= max_tokens
149+
), f"chunk has {n_tokens} tokens, exceeding max_tokens={max_tokens}"
150+
151+
152+
def test_chunk_data_does_not_over_split():
153+
# n_chunks must be the *minimum* number of max_tokens-bounded chunks. Because
154+
# consecutive chunks overlap, that count is ceil((length - overlap) / stride),
155+
# not ceil(length / stride): the latter over-counts by one just past a stride
156+
# multiple and emits an extra redundant chunk. With max_tokens=400 / overlap=64
157+
# a 673-token doc fits in 2 chunks (~369 + ~368); the loose count gave 3 of ~267.
158+
kit = _make_kit(max_seq_length = 2048, max_generation_tokens = 760, overlap = 64)
159+
max_tokens = 2048 - 760 * 2 - 128 # 400
160+
out, contents = _chunk("word " * 673, kit = kit)
161+
assert len(out) == 2, f"673-token doc should yield the minimal 2 chunks, got {len(out)}"
162+
for content in contents:
163+
n_tokens = len(content.split())
164+
assert (
165+
n_tokens <= max_tokens
166+
), f"chunk has {n_tokens} tokens, exceeding max_tokens={max_tokens}"
167+
168+
132169
if __name__ == "__main__":
133170
test_chunk_data_keeps_single_chunk_document()
134171
test_chunk_data_still_splits_long_document()
135172
test_chunk_data_empty_document_yields_no_chunks()
136173
test_chunk_data_short_document_is_not_split_into_fragments()
137174
test_chunk_data_rejects_overlap_not_smaller_than_chunk()
138175
test_chunk_data_uninitialized_error_names_real_class()
176+
test_chunk_data_chunks_do_not_exceed_max_tokens()
177+
test_chunk_data_does_not_over_split()
139178
print("OK")

‎unsloth/dataprep/synthetic.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -425,8 +425,16 @@ def chunk_data(self, filename = None):
425425
else:
426426
# length > max_tokens > overlap here, so length - overlap > 0 and the
427427
# linspace boundaries below are always non-negative.
428-
n_chunks = int(np.ceil(length / (max_tokens - self.overlap)))
429-
boundaries = np.ceil(np.linspace(0, length - self.overlap, n_chunks)).astype(int)
428+
# Consecutive chunks overlap by `overlap`, so N chunks capped at max_tokens
429+
# cover N * (max_tokens - overlap) + overlap tokens. Covering `length` needs
430+
# ceil((length - overlap) / (max_tokens - overlap)) chunks; dividing plain
431+
# length by the stride over-counts by one just past a stride multiple and
432+
# emits an extra redundant (smaller) chunk.
433+
n_chunks = int(np.ceil((length - self.overlap) / (max_tokens - self.overlap)))
434+
# Emit n_chunks ranges: pairing boundaries[:-1] with boundaries[1:] turns
435+
# N points into N-1 ranges, so linspace needs n_chunks + 1 points. Using
436+
# n_chunks points produced one fewer, oversized chunk (exceeding max_tokens).
437+
boundaries = np.ceil(np.linspace(0, length - self.overlap, n_chunks + 1)).astype(int)
430438
boundaries = np.stack((boundaries[:-1], (boundaries + self.overlap)[1:])).T
431439
boundaries = np.minimum(boundaries, length).tolist()
432440

0 commit comments

Comments
 (0)