Skip to content

Commit 3907b8d

Browse files
Reject a negative long name size in wds2idx
TarInfo.frombuf accepts a negative base-256 size, so a crafted `././@LongLink` header with a size of -512 moved the offset back to the same header and _get_data_offset looped forever. Raise a ValueError for a negative size, and for a header cut short by the end of the archive, so every iteration either moves forward by at least one block or stops. Add tests with hand-crafted long name headers for both cases, and note that skipping a long link header is defensive, since only links, which are filtered out before, have one. Signed-off-by: Max Freedom Pollard <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 274ab46 commit 3907b8d

2 files changed

Lines changed: 42 additions & 1 deletion

File tree

‎dali/test/python/reader/test_wds2idx.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import tempfile
2222
import unittest
2323

24+
from nose_utils import assert_raises
25+
2426

2527
def _load_wds2idx():
2628
script_path = Path(__file__).parents[4] / "tools" / "wds2idx.py"
@@ -123,3 +125,32 @@ def test_wds2idx_gnu_tar_path_offsets_with_gnu_long_member_names():
123125

124126
def test_wds2idx_gnu_tar_path_offsets_with_pax_long_member_names():
125127
_check_gnu_tar_path_offsets_with_long_member_names(tarfile.PAX_FORMAT)
128+
129+
130+
def _check_gnu_long_name_header_is_rejected(size, glob):
131+
wds2idx = _load_wds2idx()
132+
133+
with tempfile.TemporaryDirectory() as test_dir:
134+
tar_path = os.path.join(test_dir, "data.tar")
135+
idx_path = os.path.join(test_dir, "data.idx")
136+
# a hand-crafted `././@LongLink` header followed by the two empty end-of-archive blocks
137+
long_name_header = tarfile.TarInfo(name="././@LongLink")
138+
long_name_header.type = tarfile.GNUTYPE_LONGNAME
139+
long_name_header.size = size
140+
with open(tar_path, "wb") as archive:
141+
archive.write(long_name_header.tobuf(tarfile.GNU_FORMAT))
142+
archive.write(bytes(2 * tarfile.BLOCKSIZE))
143+
144+
with wds2idx.IndexCreator(tar_path, idx_path, verbose=False) as creator:
145+
with open(tar_path, "rb") as archive:
146+
with assert_raises(ValueError, glob=glob):
147+
creator._get_data_offset(archive, 0)
148+
149+
150+
def test_wds2idx_gnu_long_name_header_with_negative_size():
151+
# the GNU format stores a size in base-256, which can encode a negative number
152+
_check_gnu_long_name_header_is_rejected(-tarfile.BLOCKSIZE, "Negative size -512 *")
153+
154+
155+
def test_wds2idx_gnu_long_name_header_past_end_of_archive():
156+
_check_gnu_long_name_header_is_rejected(100 * tarfile.BLOCKSIZE, "Truncated tar header *")

‎tools/wds2idx.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,12 +132,22 @@ def _get_data_offset(self, archive, block):
132132
while True:
133133
archive.seek(offset)
134134
header = archive.read(self.tar_block_size)
135+
if len(header) < self.tar_block_size:
136+
raise ValueError(f"Truncated tar header at offset {offset} in {self.uri}")
135137
offset += self.tar_block_size
136-
# the type flag is stored in the byte 156 of a tar header
138+
# the type flag is stored in the byte 156 of a tar header. A long link header is only
139+
# written for links, which never reach this point, so skipping it is defensive
137140
if header[156:157] not in (tarfile.GNUTYPE_LONGNAME, tarfile.GNUTYPE_LONGLINK):
138141
return offset
139142
# skip the blocks holding the long name
140143
size = tarfile.TarInfo.frombuf(header, tarfile.ENCODING, "surrogateescape").size
144+
# a negative size (possible in the base-256 encoding) would move the offset back and
145+
# make this loop read the same header forever
146+
if size < 0:
147+
raise ValueError(
148+
f"Negative size {size} in the long name header at offset "
149+
f"{offset - self.tar_block_size} in {self.uri}"
150+
)
141151
offset += (size + self.tar_block_size - 1) // self.tar_block_size * self.tar_block_size
142152

143153
def _get_data_tarfile(self):

0 commit comments

Comments
 (0)