Skip to content

Commit 5a98058

Browse files
ARROW-16728: [Python] Switch default and deprecate use_legacy_dataset=True in ParquetDataset (#14052)
Authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent 305026f commit 5a98058

4 files changed

Lines changed: 145 additions & 60 deletions

File tree

‎python/pyarrow/parquet/core.py‎

Lines changed: 82 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
ParquetLogicalType,
4343
FileEncryptionProperties,
4444
FileDecryptionProperties)
45-
from pyarrow.fs import (LocalFileSystem, FileSystem,
45+
from pyarrow.fs import (LocalFileSystem, FileSystem, FileType,
4646
_resolve_filesystem_and_path, _ensure_filesystem)
4747
from pyarrow import filesystem as legacyfs
4848
from pyarrow.util import guid, _is_path_like, _stringify_path, _deprecate_api
@@ -1701,7 +1701,7 @@ class ParquetDataset:
17011701
dataset metadata. Increasing this is helpful to read partitioned
17021702
datasets.
17031703
{0}
1704-
use_legacy_dataset : bool, default True
1704+
use_legacy_dataset : bool, default False
17051705
Set to False to enable the new code path (using the
17061706
new Arrow Dataset API). Among other things, this allows to pass
17071707
`filters` for all columns and not only the partition keys, enables
@@ -1735,17 +1735,10 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17351735
metadata=None, split_row_groups=False, validate_schema=True,
17361736
filters=None, metadata_nthreads=None, read_dictionary=None,
17371737
memory_map=False, buffer_size=0, partitioning="hive",
1738-
use_legacy_dataset=None, pre_buffer=True,
1738+
use_legacy_dataset=False, pre_buffer=True,
17391739
coerce_int96_timestamp_unit=None,
17401740
thrift_string_size_limit=None,
17411741
thrift_container_size_limit=None):
1742-
if use_legacy_dataset is None:
1743-
# if a new filesystem is passed -> default to new implementation
1744-
if isinstance(filesystem, FileSystem):
1745-
use_legacy_dataset = False
1746-
# otherwise the default is still True
1747-
else:
1748-
use_legacy_dataset = True
17491742

17501743
if not use_legacy_dataset:
17511744
return _ParquetDatasetV2(
@@ -1765,14 +1758,19 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17651758
thrift_string_size_limit=thrift_string_size_limit,
17661759
thrift_container_size_limit=thrift_container_size_limit,
17671760
)
1761+
warnings.warn(
1762+
"Passing 'use_legacy_dataset=True' to get the legacy behaviour is "
1763+
"deprecated as of pyarrow 11.0.0, and the legacy implementation "
1764+
"will be removed in a future version.",
1765+
FutureWarning, stacklevel=2)
17681766
self = object.__new__(cls)
17691767
return self
17701768

17711769
def __init__(self, path_or_paths, filesystem=None, schema=None,
17721770
metadata=None, split_row_groups=False, validate_schema=True,
17731771
filters=None, metadata_nthreads=None, read_dictionary=None,
17741772
memory_map=False, buffer_size=0, partitioning="hive",
1775-
use_legacy_dataset=True, pre_buffer=True,
1773+
use_legacy_dataset=False, pre_buffer=True,
17761774
coerce_int96_timestamp_unit=None,
17771775
thrift_string_size_limit=None,
17781776
thrift_container_size_limit=None):
@@ -1859,6 +1857,12 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
18591857
if validate_schema:
18601858
self.validate_schemas()
18611859

1860+
def __getnewargs_ex__(self):
1861+
# when creating a new instance while unpickling, force to use the
1862+
# legacy code path to create a ParquetDataset instance
1863+
# instead of a _ParquetDatasetV2 instance
1864+
return ((), dict(use_legacy_dataset=True))
1865+
18621866
def equals(self, other):
18631867
if not isinstance(other, ParquetDataset):
18641868
raise TypeError('`other` must be an instance of ParquetDataset')
@@ -2415,6 +2419,7 @@ def __init__(self, path_or_paths, filesystem=None, *, filters=None,
24152419

24162420
# check for single fragment dataset
24172421
single_file = None
2422+
self._base_dir = None
24182423
if not isinstance(path_or_paths, list):
24192424
if _is_path_like(path_or_paths):
24202425
path_or_paths = _stringify_path(path_or_paths)
@@ -2425,8 +2430,11 @@ def __init__(self, path_or_paths, filesystem=None, *, filters=None,
24252430
path_or_paths)
24262431
except ValueError:
24272432
filesystem = LocalFileSystem(use_mmap=memory_map)
2428-
if filesystem.get_file_info(path_or_paths).is_file:
2433+
finfo = filesystem.get_file_info(path_or_paths)
2434+
if finfo.is_file:
24292435
single_file = path_or_paths
2436+
if finfo.type == FileType.Directory:
2437+
self._base_dir = path_or_paths
24302438
else:
24312439
single_file = path_or_paths
24322440

@@ -2452,6 +2460,27 @@ def __init__(self, path_or_paths, filesystem=None, *, filters=None,
24522460
partitioning=partitioning,
24532461
ignore_prefixes=ignore_prefixes)
24542462

2463+
def equals(self, other):
2464+
if isinstance(other, ParquetDataset):
2465+
raise TypeError(
2466+
"`other` must be an instance of ParquetDataset constructed "
2467+
"with `use_legacy_dataset=False`"
2468+
)
2469+
if not isinstance(other, _ParquetDatasetV2):
2470+
raise TypeError('`other` must be an instance of ParquetDataset')
2471+
2472+
return (self.schema == other.schema and
2473+
self._dataset.format == other._dataset.format and
2474+
self.filesystem == other.filesystem and
2475+
# self.fragments == other.fragments and
2476+
self.files == other.files)
2477+
2478+
def __eq__(self, other):
2479+
try:
2480+
return self.equals(other)
2481+
except TypeError:
2482+
return NotImplemented
2483+
24552484
@property
24562485
def schema(self):
24572486
"""
@@ -2529,7 +2558,16 @@ def read(self, columns=None, use_threads=True, use_pandas_metadata=False):
25292558
"""
25302559
# if use_pandas_metadata, we need to include index columns in the
25312560
# column selection, to be able to restore those in the pandas DataFrame
2532-
metadata = self.schema.metadata
2561+
metadata = self.schema.metadata or {}
2562+
2563+
if use_pandas_metadata:
2564+
# if the dataset schema metadata itself doesn't have pandas
2565+
# then try to get this from common file (for backwards compat)
2566+
if b"pandas" not in metadata:
2567+
common_metadata = self._get_common_pandas_metadata()
2568+
if common_metadata:
2569+
metadata = common_metadata
2570+
25332571
if columns is not None and use_pandas_metadata:
25342572
if metadata and b'pandas' in metadata:
25352573
# RangeIndex can be represented as dict instead of column name
@@ -2556,6 +2594,24 @@ def read(self, columns=None, use_threads=True, use_pandas_metadata=False):
25562594

25572595
return table
25582596

2597+
def _get_common_pandas_metadata(self):
2598+
2599+
if not self._base_dir:
2600+
return None
2601+
2602+
metadata = None
2603+
for name in ["_common_metadata", "_metadata"]:
2604+
metadata_path = os.path.join(str(self._base_dir), name)
2605+
finfo = self.filesystem.get_file_info(metadata_path)
2606+
if finfo.is_file:
2607+
pq_meta = read_metadata(
2608+
metadata_path, filesystem=self.filesystem)
2609+
metadata = pq_meta.metadata
2610+
if metadata and b'pandas' in metadata:
2611+
break
2612+
2613+
return metadata
2614+
25592615
def read_pandas(self, **kwargs):
25602616
"""
25612617
Read dataset including pandas metadata, if any. Other arguments passed
@@ -2915,14 +2971,19 @@ def read_table(source, *, columns=None, use_threads=True, metadata=None,
29152971
"use_legacy_dataset=False")
29162972

29172973
if _is_path_like(source):
2918-
pf = ParquetDataset(
2919-
source, metadata=metadata, memory_map=memory_map,
2920-
read_dictionary=read_dictionary,
2921-
buffer_size=buffer_size,
2922-
filesystem=filesystem, filters=filters,
2923-
partitioning=partitioning,
2924-
coerce_int96_timestamp_unit=coerce_int96_timestamp_unit
2925-
)
2974+
with warnings.catch_warnings():
2975+
# Suppress second warning from ParquetDataset constructor
2976+
warnings.filterwarnings(
2977+
"ignore", "Passing 'use_legacy_dataset", FutureWarning)
2978+
pf = ParquetDataset(
2979+
source, metadata=metadata, memory_map=memory_map,
2980+
read_dictionary=read_dictionary,
2981+
buffer_size=buffer_size,
2982+
filesystem=filesystem, filters=filters,
2983+
partitioning=partitioning,
2984+
coerce_int96_timestamp_unit=coerce_int96_timestamp_unit,
2985+
use_legacy_dataset=True,
2986+
)
29262987
else:
29272988
pf = ParquetFile(
29282989
source, metadata=metadata,

‎python/pyarrow/tests/parquet/test_basic.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ def test_fspath(tempdir, use_legacy_dataset):
291291
])
292292
@pytest.mark.parametrize("name", ("data.parquet", "例.parquet"))
293293
def test_relative_paths(tempdir, use_legacy_dataset, filesystem, name):
294+
if use_legacy_dataset and isinstance(filesystem, fs.FileSystem):
295+
pytest.skip("Passing new filesystem not supported for legacy reader")
294296
# reading and writing from relative paths
295297
table = pa.table({"a": [1, 2, 3]})
296298
path = tempdir / name

‎python/pyarrow/tests/parquet/test_dataset.py‎

Lines changed: 54 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,9 @@ def test_create_parquet_dataset_multi_threaded(tempdir):
164164
FutureWarning, match="Specifying the 'metadata_nthreads'"
165165
):
166166
dataset = pq.ParquetDataset(
167-
base_path, filesystem=fs, metadata_nthreads=16)
167+
base_path, filesystem=fs, metadata_nthreads=16,
168+
use_legacy_dataset=True
169+
)
168170
assert len(dataset.pieces) > 0
169171
partitions = dataset.partitions
170172
assert len(partitions.partition_names) > 0
@@ -266,10 +268,12 @@ def test_filters_equivalency(tempdir, use_legacy_dataset):
266268
# results with the current code.
267269
with pytest.raises(NotImplementedError):
268270
filters = [[('string', '==', b'1\0a')]]
269-
pq.ParquetDataset(base_path, filesystem=fs, filters=filters)
271+
pq.ParquetDataset(base_path, filesystem=fs, filters=filters,
272+
use_legacy_dataset=True)
270273
with pytest.raises(NotImplementedError):
271274
filters = [[('string', '==', '1\0a')]]
272-
pq.ParquetDataset(base_path, filesystem=fs, filters=filters)
275+
pq.ParquetDataset(base_path, filesystem=fs, filters=filters,
276+
use_legacy_dataset=True)
273277
else:
274278
for filters in [[[('string', '==', b'1\0a')]],
275279
[[('string', '==', '1\0a')]]]:
@@ -802,7 +806,8 @@ def _test_read_common_metadata_files(fs, base_path):
802806
with fs.open(metadata_path, 'wb') as f:
803807
pq.write_metadata(table.schema, f)
804808

805-
dataset = pq.ParquetDataset(base_path, filesystem=fs)
809+
dataset = pq.ParquetDataset(base_path, filesystem=fs,
810+
use_legacy_dataset=True)
806811
with pytest.warns(FutureWarning):
807812
assert dataset.common_metadata_path == str(metadata_path)
808813

@@ -811,7 +816,8 @@ def _test_read_common_metadata_files(fs, base_path):
811816
assert dataset.schema.equals(common_schema)
812817

813818
# handle list of one directory
814-
dataset2 = pq.ParquetDataset([base_path], filesystem=fs)
819+
dataset2 = pq.ParquetDataset([base_path], filesystem=fs,
820+
use_legacy_dataset=True)
815821
assert dataset2.schema.equals(dataset.schema)
816822

817823

@@ -844,7 +850,8 @@ def test_read_metadata_files(tempdir):
844850
with fs.open(metadata_path, 'wb') as f:
845851
pq.write_metadata(table.schema, f)
846852

847-
dataset = pq.ParquetDataset(tempdir, filesystem=fs)
853+
dataset = pq.ParquetDataset(tempdir, filesystem=fs,
854+
use_legacy_dataset=True)
848855
with pytest.warns(FutureWarning):
849856
assert dataset.metadata_path == str(metadata_path)
850857

@@ -941,7 +948,8 @@ def read_multiple_files(paths, columns=None, use_threads=True, **kwargs):
941948
assert result2.equals(expected)
942949

943950
with pytest.warns(FutureWarning, match="Specifying the 'schema'"):
944-
result3 = pq.ParquetDataset(dirpath, schema=metadata.schema).read()
951+
result3 = pq.ParquetDataset(dirpath, schema=metadata.schema,
952+
use_legacy_dataset=True).read()
945953
assert result3.equals(expected)
946954
else:
947955
with pytest.raises(ValueError, match="no longer supported"):
@@ -1481,8 +1489,7 @@ def test_write_to_dataset_filesystem(tempdir):
14811489
assert result.equals(table)
14821490

14831491

1484-
# TODO(dataset) support pickling
1485-
def _make_dataset_for_pickling(tempdir, N=100):
1492+
def _make_dataset_for_pickling(tempdir, use_legacy_dataset=False, N=100):
14861493
path = tempdir / 'data.parquet'
14871494
fs = LocalFileSystem._get_instance()
14881495

@@ -1504,46 +1511,53 @@ def _make_dataset_for_pickling(tempdir, N=100):
15041511
with fs.open(metadata_path, 'wb') as f:
15051512
pq.write_metadata(table.schema, f)
15061513

1507-
dataset = pq.ParquetDataset(tempdir, filesystem=fs)
1508-
with pytest.warns(FutureWarning):
1509-
assert dataset.metadata_path == str(metadata_path)
1514+
dataset = pq.ParquetDataset(
1515+
tempdir, filesystem=fs, use_legacy_dataset=use_legacy_dataset)
1516+
if use_legacy_dataset:
1517+
with pytest.warns(FutureWarning):
1518+
assert dataset.metadata_path == str(metadata_path)
15101519

15111520
return dataset
15121521

15131522

1514-
def _assert_dataset_is_picklable(dataset, pickler):
1523+
def _assert_dataset_is_picklable(dataset, pickler, use_legacy_dataset=False):
15151524
def is_pickleable(obj):
15161525
return obj == pickler.loads(pickler.dumps(obj))
15171526

15181527
assert is_pickleable(dataset)
1519-
with pytest.warns(FutureWarning):
1520-
metadata = dataset.metadata
1521-
assert is_pickleable(metadata)
1522-
assert is_pickleable(metadata.schema)
1523-
assert len(metadata.schema)
1524-
for column in metadata.schema:
1525-
assert is_pickleable(column)
1526-
1527-
for piece in dataset._pieces:
1528-
assert is_pickleable(piece)
1529-
metadata = piece.get_metadata()
1530-
assert metadata.num_row_groups
1531-
for i in range(metadata.num_row_groups):
1532-
assert is_pickleable(metadata.row_group(i))
1528+
if use_legacy_dataset:
1529+
with pytest.warns(FutureWarning):
1530+
metadata = dataset.metadata
1531+
assert is_pickleable(metadata)
1532+
assert is_pickleable(metadata.schema)
1533+
assert len(metadata.schema)
1534+
for column in metadata.schema:
1535+
assert is_pickleable(column)
1536+
1537+
for piece in dataset._pieces:
1538+
assert is_pickleable(piece)
1539+
metadata = piece.get_metadata()
1540+
assert metadata.num_row_groups
1541+
for i in range(metadata.num_row_groups):
1542+
assert is_pickleable(metadata.row_group(i))
15331543

15341544

15351545
@pytest.mark.pandas
1536-
def test_builtin_pickle_dataset(tempdir, datadir):
1546+
@parametrize_legacy_dataset
1547+
def test_builtin_pickle_dataset(tempdir, datadir, use_legacy_dataset):
15371548
import pickle
1538-
dataset = _make_dataset_for_pickling(tempdir)
1539-
_assert_dataset_is_picklable(dataset, pickler=pickle)
1549+
dataset = _make_dataset_for_pickling(tempdir, use_legacy_dataset)
1550+
_assert_dataset_is_picklable(
1551+
dataset, pickler=pickle, use_legacy_dataset=use_legacy_dataset)
15401552

15411553

15421554
@pytest.mark.pandas
1543-
def test_cloudpickle_dataset(tempdir, datadir):
1555+
@parametrize_legacy_dataset
1556+
def test_cloudpickle_dataset(tempdir, datadir, use_legacy_dataset):
15441557
cp = pytest.importorskip('cloudpickle')
1545-
dataset = _make_dataset_for_pickling(tempdir)
1546-
_assert_dataset_is_picklable(dataset, pickler=cp)
1558+
dataset = _make_dataset_for_pickling(tempdir, use_legacy_dataset)
1559+
_assert_dataset_is_picklable(
1560+
dataset, pickler=cp, use_legacy_dataset=use_legacy_dataset)
15471561

15481562

15491563
@pytest.mark.pandas
@@ -1699,7 +1713,10 @@ def test_parquet_dataset_new_filesystem(tempdir):
16991713

17001714

17011715
@pytest.mark.filterwarnings("ignore:'ParquetDataset:FutureWarning")
1702-
def test_parquet_dataset_partitions_piece_path_with_fsspec(tempdir):
1716+
@parametrize_legacy_dataset
1717+
def test_parquet_dataset_partitions_piece_path_with_fsspec(
1718+
tempdir, use_legacy_dataset
1719+
):
17031720
# ARROW-10462 ensure that on Windows we properly use posix-style paths
17041721
# as used by fsspec
17051722
fsspec = pytest.importorskip("fsspec")
@@ -1709,7 +1726,8 @@ def test_parquet_dataset_partitions_piece_path_with_fsspec(tempdir):
17091726

17101727
# pass a posix-style path (using "/" also on Windows)
17111728
path = str(tempdir).replace("\\", "/")
1712-
dataset = pq.ParquetDataset(path, filesystem=filesystem)
1729+
dataset = pq.ParquetDataset(
1730+
path, filesystem=filesystem, use_legacy_dataset=use_legacy_dataset)
17131731
# ensure the piece path is also posix-style
17141732
expected = path + "/data.parquet"
17151733
assert dataset.pieces[0].path == expected
@@ -1720,7 +1738,7 @@ def test_parquet_dataset_deprecated_properties(tempdir):
17201738
table = pa.table({'a': [1, 2, 3]})
17211739
path = tempdir / 'data.parquet'
17221740
pq.write_table(table, path)
1723-
dataset = pq.ParquetDataset(path)
1741+
dataset = pq.ParquetDataset(path, use_legacy_dataset=True)
17241742

17251743
with pytest.warns(FutureWarning, match="'ParquetDataset.pieces"):
17261744
dataset.pieces

0 commit comments

Comments
 (0)