@@ -1368,7 +1368,7 @@ class ParquetDataset:
13681368
13691369 def __new__ (cls , path_or_paths = None , filesystem = None , schema = None ,
13701370 metadata = None , split_row_groups = False , validate_schema = True ,
1371- filters = None , metadata_nthreads = 1 , read_dictionary = None ,
1371+ filters = None , metadata_nthreads = None , read_dictionary = None ,
13721372 memory_map = False , buffer_size = 0 , partitioning = "hive" ,
13731373 use_legacy_dataset = None , pre_buffer = True ,
13741374 coerce_int96_timestamp_unit = None ):
@@ -1401,14 +1401,24 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
14011401
14021402 def __init__ (self , path_or_paths , filesystem = None , schema = None ,
14031403 metadata = None , split_row_groups = False , validate_schema = True ,
1404- filters = None , metadata_nthreads = 1 , read_dictionary = None ,
1404+ filters = None , metadata_nthreads = None , read_dictionary = None ,
14051405 memory_map = False , buffer_size = 0 , partitioning = "hive" ,
14061406 use_legacy_dataset = True , pre_buffer = True ,
14071407 coerce_int96_timestamp_unit = None ):
14081408 if partitioning != "hive" :
14091409 raise ValueError (
14101410 'Only "hive" for hive-like partitioning is supported when '
14111411 'using use_legacy_dataset=True' )
1412+ if metadata_nthreads is not None :
1413+ warnings .warn (
1414+ "Specifying the 'metadata_nthreads' argument is deprecated as "
1415+ "of pyarrow 8.0.0, and the argument will be removed in a "
1416+ "future version" ,
1417+ DeprecationWarning , stacklevel = 2 ,
1418+ )
1419+ else :
1420+ metadata_nthreads = 1
1421+
14121422 self ._metadata = _ParquetDatasetMetadata ()
14131423 a_path = path_or_paths
14141424 if isinstance (a_path , list ):
@@ -1447,7 +1457,15 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
14471457 else :
14481458 self .metadata = metadata
14491459
1450- self .schema = schema
1460+ if schema is not None :
1461+ warnings .warn (
1462+ "Specifying the 'schema' argument with 'use_legacy_dataset="
1463+ "True' is deprecated as of pyarrow 8.0.0. You can still "
1464+ "specify it in combination with 'use_legacy_dataet=False', "
1465+ "but in that case you need to specify a pyarrow.Schema "
1466+ "instead of a ParquetSchema." ,
1467+ DeprecationWarning , stacklevel = 2 )
1468+ self ._schema = schema
14511469
14521470 self .split_row_groups = split_row_groups
14531471
@@ -1469,7 +1487,7 @@ def equals(self, other):
14691487 return False
14701488 for prop in ('paths' , '_pieces' , '_partitions' ,
14711489 'common_metadata_path' , 'metadata_path' ,
1472- 'common_metadata' , 'metadata' , 'schema ' ,
1490+ 'common_metadata' , 'metadata' , '_schema ' ,
14731491 'split_row_groups' ):
14741492 if getattr (self , prop ) != getattr (other , prop ):
14751493 return False
@@ -1486,16 +1504,16 @@ def __eq__(self, other):
14861504 return NotImplemented
14871505
14881506 def validate_schemas (self ):
1489- if self .metadata is None and self .schema is None :
1507+ if self .metadata is None and self ._schema is None :
14901508 if self .common_metadata is not None :
1491- self .schema = self .common_metadata .schema
1509+ self ._schema = self .common_metadata .schema
14921510 else :
1493- self .schema = self ._pieces [0 ].get_metadata ().schema
1494- elif self .schema is None :
1495- self .schema = self .metadata .schema
1511+ self ._schema = self ._pieces [0 ].get_metadata ().schema
1512+ elif self ._schema is None :
1513+ self ._schema = self .metadata .schema
14961514
14971515 # Verify schemas are all compatible
1498- dataset_schema = self .schema .to_arrow_schema ()
1516+ dataset_schema = self ._schema .to_arrow_schema ()
14991517 # Exclude the partition columns from the schema, they are provided
15001518 # by the path, not the DatasetPiece
15011519 if self ._partitions is not None :
@@ -1612,6 +1630,18 @@ def partitions(self):
16121630 DeprecationWarning , stacklevel = 2 )
16131631 return self ._partitions
16141632
1633+ @property
1634+ def schema (self ):
1635+ warnings .warn (
1636+ _DEPR_MSG .format (
1637+ "ParquetDataset.schema" ,
1638+ " Specify 'use_legacy_dataset=False' while constructing the "
1639+ "ParquetDataset, and then use the '.schema' attribute "
1640+ "instead (which will return an Arrow schema instead of a "
1641+ "Parquet schema)." ),
1642+ DeprecationWarning , stacklevel = 2 )
1643+ return self ._schema
1644+
16151645 @property
16161646 def memory_map (self ):
16171647 warnings .warn (
@@ -1706,15 +1736,14 @@ class _ParquetDatasetV2:
17061736 def __init__ (self , path_or_paths , filesystem = None , filters = None ,
17071737 partitioning = "hive" , read_dictionary = None , buffer_size = None ,
17081738 memory_map = False , ignore_prefixes = None , pre_buffer = True ,
1709- coerce_int96_timestamp_unit = None ,
1739+ coerce_int96_timestamp_unit = None , schema = None ,
17101740 decryption_properties = None , ** kwargs ):
17111741 import pyarrow .dataset as ds
17121742
17131743 # Raise error for not supported keywords
17141744 for keyword , default in [
1715- ("schema" , None ), ("metadata" , None ),
1716- ("split_row_groups" , False ), ("validate_schema" , True ),
1717- ("metadata_nthreads" , 1 )]:
1745+ ("metadata" , None ), ("split_row_groups" , False ),
1746+ ("validate_schema" , True ), ("metadata_nthreads" , None )]:
17181747 if keyword in kwargs and kwargs [keyword ] is not default :
17191748 raise ValueError (
17201749 "Keyword '{0}' is not yet supported with the new "
@@ -1785,7 +1814,7 @@ def __init__(self, path_or_paths, filesystem=None, filters=None,
17851814 fragment = parquet_format .make_fragment (single_file , filesystem )
17861815
17871816 self ._dataset = ds .FileSystemDataset (
1788- [fragment ], schema = fragment .physical_schema ,
1817+ [fragment ], schema = schema or fragment .physical_schema ,
17891818 format = parquet_format ,
17901819 filesystem = fragment .filesystem
17911820 )
@@ -1797,7 +1826,7 @@ def __init__(self, path_or_paths, filesystem=None, filters=None,
17971826 infer_dictionary = True )
17981827
17991828 self ._dataset = ds .dataset (path_or_paths , filesystem = filesystem ,
1800- format = parquet_format ,
1829+ schema = schema , format = parquet_format ,
18011830 partitioning = partitioning ,
18021831 ignore_prefixes = ignore_prefixes )
18031832
@@ -1909,6 +1938,9 @@ def partitioning(self):
19091938 Perform multi-threaded column reads.
19101939metadata : FileMetaData
19111940 If separately computed
1941+ schema : Schema, optional
1942+ Optionally provide the Schema for the parquet dataset, in which case it
1943+ will not be inferred from the source.
19121944{1}
19131945use_legacy_dataset : bool, default False
19141946 By default, `read_table` uses the new Arrow Datasets API since
@@ -1959,7 +1991,7 @@ def partitioning(self):
19591991
19601992
19611993def read_table (source , columns = None , use_threads = True , metadata = None ,
1962- use_pandas_metadata = False , memory_map = False ,
1994+ schema = None , use_pandas_metadata = False , memory_map = False ,
19631995 read_dictionary = None , filesystem = None , filters = None ,
19641996 buffer_size = 0 , partitioning = "hive" , use_legacy_dataset = False ,
19651997 ignore_prefixes = None , pre_buffer = True ,
@@ -1976,6 +2008,7 @@ def read_table(source, columns=None, use_threads=True, metadata=None,
19762008 try :
19772009 dataset = _ParquetDatasetV2 (
19782010 source ,
2011+ schema = schema ,
19792012 filesystem = filesystem ,
19802013 partitioning = partitioning ,
19812014 memory_map = memory_map ,
@@ -1999,6 +2032,11 @@ def read_table(source, columns=None, use_threads=True, metadata=None,
19992032 "the 'partitioning' keyword is not supported when the "
20002033 "pyarrow.dataset module is not available"
20012034 )
2035+ if schema is not None :
2036+ raise ValueError (
2037+ "the 'schema' argument is not supported when the "
2038+ "pyarrow.dataset module is not available"
2039+ )
20022040 filesystem , path = _resolve_filesystem_and_path (source , filesystem )
20032041 if filesystem is not None :
20042042 source = filesystem .open_input_file (path )
@@ -2025,6 +2063,11 @@ def read_table(source, columns=None, use_threads=True, metadata=None,
20252063 "The 'ignore_prefixes' keyword is only supported when "
20262064 "use_legacy_dataset=False" )
20272065
2066+ if schema is not None :
2067+ raise ValueError (
2068+ "The 'schema' argument is only supported when "
2069+ "use_legacy_dataset=False" )
2070+
20282071 if _is_path_like (source ):
20292072 pf = ParquetDataset (
20302073 source , metadata = metadata , memory_map = memory_map ,
0 commit comments