Skip to content

Commit 42aa468

Browse files
authored
Mirror readinto from the pyarrow stream onto ArrowFile (#2111)
1 parent 4a4e6ae commit 42aa468

2 files changed

Lines changed: 37 additions & 0 deletions

File tree

‎fsspec/implementations/arrow.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ def get_file(self, rpath, lpath, **kwargs):
227227
"stream",
228228
[
229229
"read",
230+
"readinto",
230231
"seek",
231232
"tell",
232233
"write",

‎fsspec/implementations/tests/test_arrow.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import io
12
import secrets
23

34
import pytest
@@ -306,6 +307,41 @@ def test_seekable(fs, remote_dir):
306307
file.seek(5)
307308

308309

310+
def test_readinto(fs, remote_dir):
311+
data = b"dvc.org"
312+
313+
with fs.open(remote_dir + "/a.txt", "wb") as stream:
314+
stream.write(data)
315+
316+
for seekable in [True, False]:
317+
with fs.open(remote_dir + "/a.txt", "rb", seekable=seekable) as file:
318+
buffer = bytearray(3)
319+
assert file.readinto(buffer) == 3
320+
assert bytes(buffer) == data[:3]
321+
322+
323+
@pytest.mark.parametrize(
324+
"read",
325+
[
326+
lambda buffered: buffered.read(3),
327+
lambda buffered: buffered.peek(3)[:3],
328+
lambda buffered: buffered.read1(3),
329+
],
330+
ids=["read", "peek", "read1"],
331+
)
332+
def test_readinto_supports_a_buffered_reader(fs, remote_dir, read):
333+
# Each of these fills a buffer io.BufferedReader owns, so they go through
334+
# readinto rather than read(); an unsized read() does not. A gzip reader takes
335+
# the peek()/read1() path.
336+
data = b"dvc.org"
337+
338+
with fs.open(remote_dir + "/a.txt", "wb") as stream:
339+
stream.write(data)
340+
341+
with fs.open(remote_dir + "/a.txt", "rb") as file:
342+
assert read(io.BufferedReader(file)) == data[:3]
343+
344+
309345
def test_get_kwargs_from_urls_hadoop_fs():
310346
kwargs = HadoopFileSystem._get_kwargs_from_urls(
311347
"hdfs://user@localhost:8020/?replication=2"

0 commit comments

Comments
 (0)