Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix(datasets): use len(lengths) for default modes in multimodal add_d…
…ocument

IndexedDatasetBuilder.add_document defaults modes to None, and on a multimodal
builder it then built the per-item mode list as [0] * lengths. lengths is a
List[int], so [0] * lengths multiplies a list by a list and raises TypeError
instead of producing one default mode per item. Use [0] * len(lengths), which
matches the one-mode-per-item contract that add_item and sequence_lengths
already follow.

Signed-off-by: winklemad <[email protected]>
  • Loading branch information
winklemad committed Jul 9, 2026
commit 8fd3c7f162780af3cd65ae921b2f4c2d5987e5f7
2 changes: 1 addition & 1 deletion megatron/core/datasets/indexed_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -994,7 +994,7 @@ def add_document(
self.sequence_lengths.extend(lengths)
self.document_indices.append(len(self.sequence_lengths))
if self.multimodal:
self.sequence_modes.extend(modes if modes is not None else [0] * lengths)
self.sequence_modes.extend(modes if modes is not None else [0] * len(lengths))

def end_document(self) -> None:
"""Finalize the document, for use with IndexedDatasetBuilder.add_item"""
Expand Down
21 changes: 21 additions & 0 deletions tests/unit_tests/data/test_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,27 @@ def create_file_prefixes(tokenizer, number_of_files, maximum_number_of_documents
return file_prefixes


def test_multimodal_builder_add_document_default_modes():
# add_document documents modes as defaulting to None. On a multimodal builder
# that default must assign one mode (0) per item, matching the sequence
# lengths - not crash. Regression test for `[0] * lengths` (list * list).
with tempfile.TemporaryDirectory() as temp_dir:
builder = IndexedDatasetBuilder(
os.path.join(temp_dir, "mm.bin"), dtype=numpy.int32, multimodal=True
)

builder.add_document(numpy.array([1, 2, 3, 4, 5], dtype=numpy.int32), [3, 2])
builder.add_document(numpy.array([6, 7], dtype=numpy.int32), [2])

assert builder.sequence_lengths == [3, 2, 2]
assert builder.sequence_modes == [0, 0, 0]
assert len(builder.sequence_modes) == len(builder.sequence_lengths)

# Explicit modes are still honored.
builder.add_document(numpy.array([8], dtype=numpy.int32), [1], modes=[4])
assert builder.sequence_modes == [0, 0, 0, 4]


def do_setup(odir):
paths = defaultdict(list)

Expand Down
Loading