Skip to content

Commit 27ae334

Browse files
ARROW-16728: [Python] ParquetDataset to still take legacy code path when old filesystem is passed
1 parent 37f5a35 commit 27ae334

1 file changed

Lines changed: 15 additions & 1 deletion

File tree

‎python/pyarrow/parquet/core.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1745,6 +1745,20 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17451745
thrift_string_size_limit=None,
17461746
thrift_container_size_limit=None):
17471747

1748+
extra_msg = ""
1749+
if use_legacy_dataset is None:
1750+
# if an old filesystem is passed -> still use to old implementation
1751+
if isinstance(filesystem, legacyfs.FileSystem):
1752+
use_legacy_dataset = True
1753+
extra_msg = (
1754+
" The legacy behaviour was still chosen because a "
1755+
"deprecated 'pyarrow.filesystem' filesystem was specified "
1756+
"(use the filesystems from pyarrow.fs instead)."
1757+
)
1758+
# otherwise the default is already False
1759+
else:
1760+
use_legacy_dataset = False
1761+
17481762
if not use_legacy_dataset:
17491763
return _ParquetDatasetV2(
17501764
path_or_paths, filesystem=filesystem,
@@ -1766,7 +1780,7 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17661780
warnings.warn(
17671781
"Passing 'use_legacy_dataset=True' to get the legacy behaviour is "
17681782
"deprecated as of pyarrow 11.0.0, and the legacy implementation "
1769-
"will be removed in a future version.",
1783+
"will be removed in a future version." + extra_msg,
17701784
FutureWarning, stacklevel=2)
17711785
self = object.__new__(cls)
17721786
return self

0 commit comments

Comments
 (0)