Skip to content

Commit 5f3df12

Browse files
ARROW-16728: [Python] ParquetDataset to still take legacy code path when old filesystem is passed (#15269)
Authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
1 parent 0121ae7 commit 5f3df12

1 file changed

Lines changed: 17 additions & 3 deletions

File tree

‎python/pyarrow/parquet/core.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,11 +1740,25 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17401740
metadata=None, split_row_groups=False, validate_schema=True,
17411741
filters=None, metadata_nthreads=None, read_dictionary=None,
17421742
memory_map=False, buffer_size=0, partitioning="hive",
1743-
use_legacy_dataset=False, pre_buffer=True,
1743+
use_legacy_dataset=None, pre_buffer=True,
17441744
coerce_int96_timestamp_unit=None,
17451745
thrift_string_size_limit=None,
17461746
thrift_container_size_limit=None):
17471747

1748+
extra_msg = ""
1749+
if use_legacy_dataset is None:
1750+
# if an old filesystem is passed -> still use to old implementation
1751+
if isinstance(filesystem, legacyfs.FileSystem):
1752+
use_legacy_dataset = True
1753+
extra_msg = (
1754+
" The legacy behaviour was still chosen because a "
1755+
"deprecated 'pyarrow.filesystem' filesystem was specified "
1756+
"(use the filesystems from pyarrow.fs instead)."
1757+
)
1758+
# otherwise the default is already False
1759+
else:
1760+
use_legacy_dataset = False
1761+
17481762
if not use_legacy_dataset:
17491763
return _ParquetDatasetV2(
17501764
path_or_paths, filesystem=filesystem,
@@ -1766,7 +1780,7 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17661780
warnings.warn(
17671781
"Passing 'use_legacy_dataset=True' to get the legacy behaviour is "
17681782
"deprecated as of pyarrow 11.0.0, and the legacy implementation "
1769-
"will be removed in a future version.",
1783+
"will be removed in a future version." + extra_msg,
17701784
FutureWarning, stacklevel=2)
17711785
self = object.__new__(cls)
17721786
return self
@@ -1775,7 +1789,7 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
17751789
metadata=None, split_row_groups=False, validate_schema=True,
17761790
filters=None, metadata_nthreads=None, read_dictionary=None,
17771791
memory_map=False, buffer_size=0, partitioning="hive",
1778-
use_legacy_dataset=False, pre_buffer=True,
1792+
use_legacy_dataset=None, pre_buffer=True,
17791793
coerce_int96_timestamp_unit=None,
17801794
thrift_string_size_limit=None,
17811795
thrift_container_size_limit=None):

0 commit comments

Comments
 (0)