Skip to content

Commit 42ed37e

Browse files
ARROW-16719: [Python] Add path/URI + filesystem handling to parquet.read_metadata (#13629)
Add `filesystem` support to `pq.read_metadata` and `pq.read_schema`. Lead-authored-by: kshitij12345 <[email protected]> Co-authored-by: Kshiteej K <[email protected]> Co-authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent f6127fc commit 42ed37e

2 files changed

Lines changed: 69 additions & 7 deletions

File tree

‎python/pyarrow/parquet/core.py‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
from collections import defaultdict
2020
from concurrent import futures
21+
from contextlib import nullcontext
2122
from functools import partial, reduce
2223

2324
import sys
@@ -3389,7 +3390,8 @@ def write_metadata(schema, where, metadata_collector=None, **kwargs):
33893390
metadata.write_metadata_file(where)
33903391

33913392

3392-
def read_metadata(where, memory_map=False, decryption_properties=None):
3393+
def read_metadata(where, memory_map=False, decryption_properties=None,
3394+
filesystem=None):
33933395
"""
33943396
Read FileMetaData from footer of a single Parquet file.
33953397
@@ -3400,6 +3402,10 @@ def read_metadata(where, memory_map=False, decryption_properties=None):
34003402
Create memory map when the source is a file path.
34013403
decryption_properties : FileDecryptionProperties, default None
34023404
Decryption properties for reading encrypted Parquet files.
3405+
filesystem : FileSystem, default None
3406+
If nothing passed, will be inferred based on path.
3407+
Path will try to be found in the local on-disk filesystem otherwise
3408+
it will be parsed as an URI to determine the filesystem.
34033409
34043410
Returns
34053411
-------
@@ -3422,11 +3428,19 @@ def read_metadata(where, memory_map=False, decryption_properties=None):
34223428
format_version: 2.6
34233429
serialized_size: ...
34243430
"""
3425-
return ParquetFile(where, memory_map=memory_map,
3426-
decryption_properties=decryption_properties).metadata
3431+
filesystem, where = _resolve_filesystem_and_path(where, filesystem)
3432+
file_ctx = nullcontext()
3433+
if filesystem is not None:
3434+
file_ctx = where = filesystem.open_input_file(where)
3435+
3436+
with file_ctx:
3437+
file = ParquetFile(where, memory_map=memory_map,
3438+
decryption_properties=decryption_properties)
3439+
return file.metadata
34273440

34283441

3429-
def read_schema(where, memory_map=False, decryption_properties=None):
3442+
def read_schema(where, memory_map=False, decryption_properties=None,
3443+
filesystem=None):
34303444
"""
34313445
Read effective Arrow schema from Parquet file metadata.
34323446
@@ -3437,6 +3451,10 @@ def read_schema(where, memory_map=False, decryption_properties=None):
34373451
Create memory map when the source is a file path.
34383452
decryption_properties : FileDecryptionProperties, default None
34393453
Decryption properties for reading encrypted Parquet files.
3454+
filesystem : FileSystem, default None
3455+
If nothing passed, will be inferred based on path.
3456+
Path will try to be found in the local on-disk filesystem otherwise
3457+
it will be parsed as an URI to determine the filesystem.
34403458
34413459
Returns
34423460
-------
@@ -3454,9 +3472,16 @@ def read_schema(where, memory_map=False, decryption_properties=None):
34543472
n_legs: int64
34553473
animal: string
34563474
"""
3457-
return ParquetFile(
3458-
where, memory_map=memory_map,
3459-
decryption_properties=decryption_properties).schema.to_arrow_schema()
3475+
filesystem, where = _resolve_filesystem_and_path(where, filesystem)
3476+
file_ctx = nullcontext()
3477+
if filesystem is not None:
3478+
file_ctx = where = filesystem.open_input_file(where)
3479+
3480+
with file_ctx:
3481+
file = ParquetFile(
3482+
where, memory_map=memory_map,
3483+
decryption_properties=decryption_properties)
3484+
return file.schema.to_arrow_schema()
34603485

34613486

34623487
# re-export everything

‎python/pyarrow/tests/parquet/test_metadata.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
import pyarrow as pa
2626
from pyarrow.tests.parquet.common import _check_roundtrip, make_sample_file
27+
from pyarrow.fs import LocalFileSystem
28+
from pyarrow.tests import util
2729

2830
try:
2931
import pyarrow.parquet as pq
@@ -533,6 +535,41 @@ def test_metadata_exceeds_message_size():
533535
metadata = pq.read_metadata(pa.BufferReader(buf))
534536

535537

538+
def test_metadata_schema_filesystem(tempdir):
539+
table = pa.table({"a": [1, 2, 3]})
540+
541+
# URI writing to local file.
542+
fname = "data.parquet"
543+
file_path = str(tempdir / fname)
544+
file_uri = 'file:///' + file_path
545+
546+
pq.write_table(table, file_path)
547+
548+
# Get expected `metadata` from path.
549+
metadata = pq.read_metadata(tempdir / fname)
550+
schema = table.schema
551+
552+
assert pq.read_metadata(file_uri).equals(metadata)
553+
assert pq.read_metadata(
554+
file_path, filesystem=LocalFileSystem()).equals(metadata)
555+
assert pq.read_metadata(
556+
fname, filesystem=f'file:///{tempdir}').equals(metadata)
557+
558+
assert pq.read_schema(file_uri).equals(schema)
559+
assert pq.read_schema(
560+
file_path, filesystem=LocalFileSystem()).equals(schema)
561+
assert pq.read_schema(
562+
fname, filesystem=f'file:///{tempdir}').equals(schema)
563+
564+
with util.change_cwd(tempdir):
565+
# Pass `filesystem` arg
566+
assert pq.read_metadata(
567+
fname, filesystem=LocalFileSystem()).equals(metadata)
568+
569+
assert pq.read_schema(
570+
fname, filesystem=LocalFileSystem()).equals(schema)
571+
572+
536573
def test_metadata_equals():
537574
table = pa.table({"a": [1, 2, 3]})
538575
with pa.BufferOutputStream() as out:

0 commit comments

Comments
 (0)