4242 ParquetLogicalType ,
4343 FileEncryptionProperties ,
4444 FileDecryptionProperties )
45- from pyarrow .fs import (LocalFileSystem , FileSystem ,
45+ from pyarrow .fs import (LocalFileSystem , FileSystem , FileType ,
4646 _resolve_filesystem_and_path , _ensure_filesystem )
4747from pyarrow import filesystem as legacyfs
4848from pyarrow .util import guid , _is_path_like , _stringify_path , _deprecate_api
@@ -1701,7 +1701,7 @@ class ParquetDataset:
17011701 dataset metadata. Increasing this is helpful to read partitioned
17021702 datasets.
17031703{0}
1704- use_legacy_dataset : bool, default True
1704+ use_legacy_dataset : bool, default False
17051705 Set to False to enable the new code path (using the
17061706 new Arrow Dataset API). Among other things, this allows to pass
17071707 `filters` for all columns and not only the partition keys, enables
@@ -1735,17 +1735,10 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17351735 metadata = None , split_row_groups = False , validate_schema = True ,
17361736 filters = None , metadata_nthreads = None , read_dictionary = None ,
17371737 memory_map = False , buffer_size = 0 , partitioning = "hive" ,
1738- use_legacy_dataset = None , pre_buffer = True ,
1738+ use_legacy_dataset = False , pre_buffer = True ,
17391739 coerce_int96_timestamp_unit = None ,
17401740 thrift_string_size_limit = None ,
17411741 thrift_container_size_limit = None ):
1742- if use_legacy_dataset is None :
1743- # if a new filesystem is passed -> default to new implementation
1744- if isinstance (filesystem , FileSystem ):
1745- use_legacy_dataset = False
1746- # otherwise the default is still True
1747- else :
1748- use_legacy_dataset = True
17491742
17501743 if not use_legacy_dataset :
17511744 return _ParquetDatasetV2 (
@@ -1765,14 +1758,19 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
17651758 thrift_string_size_limit = thrift_string_size_limit ,
17661759 thrift_container_size_limit = thrift_container_size_limit ,
17671760 )
1761+ warnings .warn (
1762+ "Passing 'use_legacy_dataset=True' to get the legacy behaviour is "
1763+ "deprecated as of pyarrow 11.0.0, and the legacy implementation "
1764+ "will be removed in a future version." ,
1765+ FutureWarning , stacklevel = 2 )
17681766 self = object .__new__ (cls )
17691767 return self
17701768
17711769 def __init__ (self , path_or_paths , filesystem = None , schema = None ,
17721770 metadata = None , split_row_groups = False , validate_schema = True ,
17731771 filters = None , metadata_nthreads = None , read_dictionary = None ,
17741772 memory_map = False , buffer_size = 0 , partitioning = "hive" ,
1775- use_legacy_dataset = True , pre_buffer = True ,
1773+ use_legacy_dataset = False , pre_buffer = True ,
17761774 coerce_int96_timestamp_unit = None ,
17771775 thrift_string_size_limit = None ,
17781776 thrift_container_size_limit = None ):
@@ -1859,6 +1857,12 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
18591857 if validate_schema :
18601858 self .validate_schemas ()
18611859
1860+ def __getnewargs_ex__ (self ):
1861+ # when creating a new instance while unpickling, force to use the
1862+ # legacy code path to create a ParquetDataset instance
1863+ # instead of a _ParquetDatasetV2 instance
1864+ return ((), dict (use_legacy_dataset = True ))
1865+
18621866 def equals (self , other ):
18631867 if not isinstance (other , ParquetDataset ):
18641868 raise TypeError ('`other` must be an instance of ParquetDataset' )
@@ -2415,6 +2419,7 @@ def __init__(self, path_or_paths, filesystem=None, *, filters=None,
24152419
24162420 # check for single fragment dataset
24172421 single_file = None
2422+ self ._base_dir = None
24182423 if not isinstance (path_or_paths , list ):
24192424 if _is_path_like (path_or_paths ):
24202425 path_or_paths = _stringify_path (path_or_paths )
@@ -2425,8 +2430,11 @@ def __init__(self, path_or_paths, filesystem=None, *, filters=None,
24252430 path_or_paths )
24262431 except ValueError :
24272432 filesystem = LocalFileSystem (use_mmap = memory_map )
2428- if filesystem .get_file_info (path_or_paths ).is_file :
2433+ finfo = filesystem .get_file_info (path_or_paths )
2434+ if finfo .is_file :
24292435 single_file = path_or_paths
2436+ if finfo .type == FileType .Directory :
2437+ self ._base_dir = path_or_paths
24302438 else :
24312439 single_file = path_or_paths
24322440
@@ -2452,6 +2460,27 @@ def __init__(self, path_or_paths, filesystem=None, *, filters=None,
24522460 partitioning = partitioning ,
24532461 ignore_prefixes = ignore_prefixes )
24542462
2463+ def equals (self , other ):
2464+ if isinstance (other , ParquetDataset ):
2465+ raise TypeError (
2466+ "`other` must be an instance of ParquetDataset constructed "
2467+ "with `use_legacy_dataset=False`"
2468+ )
2469+ if not isinstance (other , _ParquetDatasetV2 ):
2470+ raise TypeError ('`other` must be an instance of ParquetDataset' )
2471+
2472+ return (self .schema == other .schema and
2473+ self ._dataset .format == other ._dataset .format and
2474+ self .filesystem == other .filesystem and
2475+ # self.fragments == other.fragments and
2476+ self .files == other .files )
2477+
2478+ def __eq__ (self , other ):
2479+ try :
2480+ return self .equals (other )
2481+ except TypeError :
2482+ return NotImplemented
2483+
24552484 @property
24562485 def schema (self ):
24572486 """
@@ -2529,7 +2558,16 @@ def read(self, columns=None, use_threads=True, use_pandas_metadata=False):
25292558 """
25302559 # if use_pandas_metadata, we need to include index columns in the
25312560 # column selection, to be able to restore those in the pandas DataFrame
2532- metadata = self .schema .metadata
2561+ metadata = self .schema .metadata or {}
2562+
2563+ if use_pandas_metadata :
2564+ # if the dataset schema metadata itself doesn't have pandas
2565+ # then try to get this from common file (for backwards compat)
2566+ if b"pandas" not in metadata :
2567+ common_metadata = self ._get_common_pandas_metadata ()
2568+ if common_metadata :
2569+ metadata = common_metadata
2570+
25332571 if columns is not None and use_pandas_metadata :
25342572 if metadata and b'pandas' in metadata :
25352573 # RangeIndex can be represented as dict instead of column name
@@ -2556,6 +2594,24 @@ def read(self, columns=None, use_threads=True, use_pandas_metadata=False):
25562594
25572595 return table
25582596
2597+ def _get_common_pandas_metadata (self ):
2598+
2599+ if not self ._base_dir :
2600+ return None
2601+
2602+ metadata = None
2603+ for name in ["_common_metadata" , "_metadata" ]:
2604+ metadata_path = os .path .join (str (self ._base_dir ), name )
2605+ finfo = self .filesystem .get_file_info (metadata_path )
2606+ if finfo .is_file :
2607+ pq_meta = read_metadata (
2608+ metadata_path , filesystem = self .filesystem )
2609+ metadata = pq_meta .metadata
2610+ if metadata and b'pandas' in metadata :
2611+ break
2612+
2613+ return metadata
2614+
25592615 def read_pandas (self , ** kwargs ):
25602616 """
25612617 Read dataset including pandas metadata, if any. Other arguments passed
@@ -2915,14 +2971,19 @@ def read_table(source, *, columns=None, use_threads=True, metadata=None,
29152971 "use_legacy_dataset=False" )
29162972
29172973 if _is_path_like (source ):
2918- pf = ParquetDataset (
2919- source , metadata = metadata , memory_map = memory_map ,
2920- read_dictionary = read_dictionary ,
2921- buffer_size = buffer_size ,
2922- filesystem = filesystem , filters = filters ,
2923- partitioning = partitioning ,
2924- coerce_int96_timestamp_unit = coerce_int96_timestamp_unit
2925- )
2974+ with warnings .catch_warnings ():
2975+ # Suppress second warning from ParquetDataset constructor
2976+ warnings .filterwarnings (
2977+ "ignore" , "Passing 'use_legacy_dataset" , FutureWarning )
2978+ pf = ParquetDataset (
2979+ source , metadata = metadata , memory_map = memory_map ,
2980+ read_dictionary = read_dictionary ,
2981+ buffer_size = buffer_size ,
2982+ filesystem = filesystem , filters = filters ,
2983+ partitioning = partitioning ,
2984+ coerce_int96_timestamp_unit = coerce_int96_timestamp_unit ,
2985+ use_legacy_dataset = True ,
2986+ )
29262987 else :
29272988 pf = ParquetFile (
29282989 source , metadata = metadata ,
0 commit comments