Skip to content

Commit ea4ab77

Browse files
jorisvandenbosschepitrou
authored andcommitted
ARROW-15871: [Python] Start raising deprecation warnings for ParquetDataset keywords that won't be supported with the new API
Added warnings for when passing the `schema` and `metadata_nthreads` keywords. For `schema`, also enabled this keyword for the new implementation (passing through the `schema` keyword to the datasets API), but so in this case this is about a pyarrow.Schema and not a ParquetSchema that should be passed. Closes #12585 from jorisvandenbossche/ARROW-15871 Authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent ba04e7f commit ea4ab77

2 files changed

Lines changed: 111 additions & 24 deletions

File tree

‎python/pyarrow/parquet.py‎

Lines changed: 60 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1368,7 +1368,7 @@ class ParquetDataset:
13681368

13691369
def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
13701370
metadata=None, split_row_groups=False, validate_schema=True,
1371-
filters=None, metadata_nthreads=1, read_dictionary=None,
1371+
filters=None, metadata_nthreads=None, read_dictionary=None,
13721372
memory_map=False, buffer_size=0, partitioning="hive",
13731373
use_legacy_dataset=None, pre_buffer=True,
13741374
coerce_int96_timestamp_unit=None):
@@ -1401,14 +1401,24 @@ def __new__(cls, path_or_paths=None, filesystem=None, schema=None,
14011401

14021402
def __init__(self, path_or_paths, filesystem=None, schema=None,
14031403
metadata=None, split_row_groups=False, validate_schema=True,
1404-
filters=None, metadata_nthreads=1, read_dictionary=None,
1404+
filters=None, metadata_nthreads=None, read_dictionary=None,
14051405
memory_map=False, buffer_size=0, partitioning="hive",
14061406
use_legacy_dataset=True, pre_buffer=True,
14071407
coerce_int96_timestamp_unit=None):
14081408
if partitioning != "hive":
14091409
raise ValueError(
14101410
'Only "hive" for hive-like partitioning is supported when '
14111411
'using use_legacy_dataset=True')
1412+
if metadata_nthreads is not None:
1413+
warnings.warn(
1414+
"Specifying the 'metadata_nthreads' argument is deprecated as "
1415+
"of pyarrow 8.0.0, and the argument will be removed in a "
1416+
"future version",
1417+
DeprecationWarning, stacklevel=2,
1418+
)
1419+
else:
1420+
metadata_nthreads = 1
1421+
14121422
self._metadata = _ParquetDatasetMetadata()
14131423
a_path = path_or_paths
14141424
if isinstance(a_path, list):
@@ -1447,7 +1457,15 @@ def __init__(self, path_or_paths, filesystem=None, schema=None,
14471457
else:
14481458
self.metadata = metadata
14491459

1450-
self.schema = schema
1460+
if schema is not None:
1461+
warnings.warn(
1462+
"Specifying the 'schema' argument with 'use_legacy_dataset="
1463+
"True' is deprecated as of pyarrow 8.0.0. You can still "
1464+
"specify it in combination with 'use_legacy_dataet=False', "
1465+
"but in that case you need to specify a pyarrow.Schema "
1466+
"instead of a ParquetSchema.",
1467+
DeprecationWarning, stacklevel=2)
1468+
self._schema = schema
14511469

14521470
self.split_row_groups = split_row_groups
14531471

@@ -1469,7 +1487,7 @@ def equals(self, other):
14691487
return False
14701488
for prop in ('paths', '_pieces', '_partitions',
14711489
'common_metadata_path', 'metadata_path',
1472-
'common_metadata', 'metadata', 'schema',
1490+
'common_metadata', 'metadata', '_schema',
14731491
'split_row_groups'):
14741492
if getattr(self, prop) != getattr(other, prop):
14751493
return False
@@ -1486,16 +1504,16 @@ def __eq__(self, other):
14861504
return NotImplemented
14871505

14881506
def validate_schemas(self):
1489-
if self.metadata is None and self.schema is None:
1507+
if self.metadata is None and self._schema is None:
14901508
if self.common_metadata is not None:
1491-
self.schema = self.common_metadata.schema
1509+
self._schema = self.common_metadata.schema
14921510
else:
1493-
self.schema = self._pieces[0].get_metadata().schema
1494-
elif self.schema is None:
1495-
self.schema = self.metadata.schema
1511+
self._schema = self._pieces[0].get_metadata().schema
1512+
elif self._schema is None:
1513+
self._schema = self.metadata.schema
14961514

14971515
# Verify schemas are all compatible
1498-
dataset_schema = self.schema.to_arrow_schema()
1516+
dataset_schema = self._schema.to_arrow_schema()
14991517
# Exclude the partition columns from the schema, they are provided
15001518
# by the path, not the DatasetPiece
15011519
if self._partitions is not None:
@@ -1612,6 +1630,18 @@ def partitions(self):
16121630
DeprecationWarning, stacklevel=2)
16131631
return self._partitions
16141632

1633+
@property
1634+
def schema(self):
1635+
warnings.warn(
1636+
_DEPR_MSG.format(
1637+
"ParquetDataset.schema",
1638+
" Specify 'use_legacy_dataset=False' while constructing the "
1639+
"ParquetDataset, and then use the '.schema' attribute "
1640+
"instead (which will return an Arrow schema instead of a "
1641+
"Parquet schema)."),
1642+
DeprecationWarning, stacklevel=2)
1643+
return self._schema
1644+
16151645
@property
16161646
def memory_map(self):
16171647
warnings.warn(
@@ -1706,15 +1736,14 @@ class _ParquetDatasetV2:
17061736
def __init__(self, path_or_paths, filesystem=None, filters=None,
17071737
partitioning="hive", read_dictionary=None, buffer_size=None,
17081738
memory_map=False, ignore_prefixes=None, pre_buffer=True,
1709-
coerce_int96_timestamp_unit=None,
1739+
coerce_int96_timestamp_unit=None, schema=None,
17101740
decryption_properties=None, **kwargs):
17111741
import pyarrow.dataset as ds
17121742

17131743
# Raise error for not supported keywords
17141744
for keyword, default in [
1715-
("schema", None), ("metadata", None),
1716-
("split_row_groups", False), ("validate_schema", True),
1717-
("metadata_nthreads", 1)]:
1745+
("metadata", None), ("split_row_groups", False),
1746+
("validate_schema", True), ("metadata_nthreads", None)]:
17181747
if keyword in kwargs and kwargs[keyword] is not default:
17191748
raise ValueError(
17201749
"Keyword '{0}' is not yet supported with the new "
@@ -1785,7 +1814,7 @@ def __init__(self, path_or_paths, filesystem=None, filters=None,
17851814
fragment = parquet_format.make_fragment(single_file, filesystem)
17861815

17871816
self._dataset = ds.FileSystemDataset(
1788-
[fragment], schema=fragment.physical_schema,
1817+
[fragment], schema=schema or fragment.physical_schema,
17891818
format=parquet_format,
17901819
filesystem=fragment.filesystem
17911820
)
@@ -1797,7 +1826,7 @@ def __init__(self, path_or_paths, filesystem=None, filters=None,
17971826
infer_dictionary=True)
17981827

17991828
self._dataset = ds.dataset(path_or_paths, filesystem=filesystem,
1800-
format=parquet_format,
1829+
schema=schema, format=parquet_format,
18011830
partitioning=partitioning,
18021831
ignore_prefixes=ignore_prefixes)
18031832

@@ -1909,6 +1938,9 @@ def partitioning(self):
19091938
Perform multi-threaded column reads.
19101939
metadata : FileMetaData
19111940
If separately computed
1941+
schema : Schema, optional
1942+
Optionally provide the Schema for the parquet dataset, in which case it
1943+
will not be inferred from the source.
19121944
{1}
19131945
use_legacy_dataset : bool, default False
19141946
By default, `read_table` uses the new Arrow Datasets API since
@@ -1959,7 +1991,7 @@ def partitioning(self):
19591991

19601992

19611993
def read_table(source, columns=None, use_threads=True, metadata=None,
1962-
use_pandas_metadata=False, memory_map=False,
1994+
schema=None, use_pandas_metadata=False, memory_map=False,
19631995
read_dictionary=None, filesystem=None, filters=None,
19641996
buffer_size=0, partitioning="hive", use_legacy_dataset=False,
19651997
ignore_prefixes=None, pre_buffer=True,
@@ -1976,6 +2008,7 @@ def read_table(source, columns=None, use_threads=True, metadata=None,
19762008
try:
19772009
dataset = _ParquetDatasetV2(
19782010
source,
2011+
schema=schema,
19792012
filesystem=filesystem,
19802013
partitioning=partitioning,
19812014
memory_map=memory_map,
@@ -1999,6 +2032,11 @@ def read_table(source, columns=None, use_threads=True, metadata=None,
19992032
"the 'partitioning' keyword is not supported when the "
20002033
"pyarrow.dataset module is not available"
20012034
)
2035+
if schema is not None:
2036+
raise ValueError(
2037+
"the 'schema' argument is not supported when the "
2038+
"pyarrow.dataset module is not available"
2039+
)
20022040
filesystem, path = _resolve_filesystem_and_path(source, filesystem)
20032041
if filesystem is not None:
20042042
source = filesystem.open_input_file(path)
@@ -2025,6 +2063,11 @@ def read_table(source, columns=None, use_threads=True, metadata=None,
20252063
"The 'ignore_prefixes' keyword is only supported when "
20262064
"use_legacy_dataset=False")
20272065

2066+
if schema is not None:
2067+
raise ValueError(
2068+
"The 'schema' argument is only supported when "
2069+
"use_legacy_dataset=False")
2070+
20282071
if _is_path_like(source):
20292072
pf = ParquetDataset(
20302073
source, metadata=metadata, memory_map=memory_map,

‎python/pyarrow/tests/parquet/test_dataset.py‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,11 @@ def test_create_parquet_dataset_multi_threaded(tempdir):
150150

151151
manifest = pq.ParquetManifest(base_path, filesystem=fs,
152152
metadata_nthreads=1)
153-
dataset = pq.ParquetDataset(base_path, filesystem=fs, metadata_nthreads=16)
153+
with pytest.warns(
154+
DeprecationWarning, match="Specifying the 'metadata_nthreads'"
155+
):
156+
dataset = pq.ParquetDataset(
157+
base_path, filesystem=fs, metadata_nthreads=16)
154158
assert len(dataset.pieces) > 0
155159
partitions = dataset.partitions
156160
assert len(partitions.partition_names) > 0
@@ -783,12 +787,14 @@ def _test_read_common_metadata_files(fs, base_path):
783787

784788

785789
@pytest.mark.pandas
790+
@pytest.mark.filterwarnings("ignore:'ParquetDataset.schema:DeprecationWarning")
786791
def test_read_common_metadata_files(tempdir):
787792
fs = LocalFileSystem._get_instance()
788793
_test_read_common_metadata_files(fs, tempdir)
789794

790795

791796
@pytest.mark.pandas
797+
@pytest.mark.filterwarnings("ignore:'ParquetDataset.schema:DeprecationWarning")
792798
def test_read_metadata_files(tempdir):
793799
fs = LocalFileSystem._get_instance()
794800

@@ -902,7 +908,8 @@ def read_multiple_files(paths, columns=None, use_threads=True, **kwargs):
902908
result2 = read_multiple_files(paths, metadata=metadata)
903909
assert result2.equals(expected)
904910

905-
result3 = pq.ParquetDataset(dirpath, schema=metadata.schema).read()
911+
with pytest.warns(DeprecationWarning, match="Specifying the 'schema'"):
912+
result3 = pq.ParquetDataset(dirpath, schema=metadata.schema).read()
906913
assert result3.equals(expected)
907914
else:
908915
with pytest.raises(ValueError, match="no longer supported"):
@@ -947,7 +954,8 @@ def read_multiple_files(paths, columns=None, use_threads=True, **kwargs):
947954
mixed_paths = [bad_apple_path, paths[0]]
948955

949956
with pytest.raises(ValueError):
950-
read_multiple_files(mixed_paths, schema=bad_meta.schema)
957+
with pytest.warns(DeprecationWarning, match="Specifying the 'schema'"):
958+
read_multiple_files(mixed_paths, schema=bad_meta.schema)
951959

952960
with pytest.raises(ValueError):
953961
read_multiple_files(mixed_paths)
@@ -1195,6 +1203,7 @@ def test_empty_directory(tempdir, use_legacy_dataset):
11951203
assert result.num_columns == 0
11961204

11971205

1206+
@pytest.mark.filterwarnings("ignore:'ParquetDataset.schema:DeprecationWarning")
11981207
def _test_write_to_dataset_with_partitions(base_path,
11991208
use_legacy_dataset=True,
12001209
filesystem=None,
@@ -1236,7 +1245,8 @@ def _test_write_to_dataset_with_partitions(base_path,
12361245
use_legacy_dataset=use_legacy_dataset)
12371246
# ARROW-2209: Ensure the dataset schema also includes the partition columns
12381247
if use_legacy_dataset:
1239-
dataset_cols = set(dataset.schema.to_arrow_schema().names)
1248+
with pytest.warns(DeprecationWarning, match="'ParquetDataset.schema'"):
1249+
dataset_cols = set(dataset.schema.to_arrow_schema().names)
12401250
else:
12411251
# NB schema property is an arrow and not parquet schema
12421252
dataset_cols = set(dataset.schema.names)
@@ -1541,12 +1551,43 @@ def test_dataset_read_dictionary(tempdir, use_legacy_dataset):
15411551
assert c1.equals(ex_chunks[0])
15421552

15431553

1554+
@pytest.mark.pandas
1555+
def test_read_table_schema(tempdir):
1556+
# test that schema keyword is passed through in read_table
1557+
table = pa.table({'a': pa.array([1, 2, 3], pa.int32())})
1558+
pq.write_table(table, tempdir / "data1.parquet")
1559+
pq.write_table(table, tempdir / "data2.parquet")
1560+
1561+
schema = pa.schema([('a', 'int64')])
1562+
1563+
# reading single file (which is special cased in the code)
1564+
result = pq.read_table(tempdir / "data1.parquet", schema=schema)
1565+
expected = pa.table({'a': [1, 2, 3]}, schema=schema)
1566+
assert result.equals(expected)
1567+
1568+
# reading multiple fiels
1569+
result = pq.read_table(tempdir, schema=schema)
1570+
expected = pa.table({'a': [1, 2, 3, 1, 2, 3]}, schema=schema)
1571+
assert result.equals(expected)
1572+
1573+
# don't allow it with the legacy reader
1574+
with pytest.raises(
1575+
ValueError, match="The 'schema' argument is only supported"
1576+
):
1577+
pq.read_table(tempdir / "data.parquet", schema=schema,
1578+
use_legacy_dataset=True)
1579+
1580+
# using ParquetDataset directory with non-legacy implementation
1581+
result = pq.ParquetDataset(
1582+
tempdir, schema=schema, use_legacy_dataset=False
1583+
)
1584+
expected = pa.table({'a': [1, 2, 3, 1, 2, 3]}, schema=schema)
1585+
assert result.read().equals(expected)
1586+
1587+
15441588
@pytest.mark.dataset
15451589
def test_dataset_unsupported_keywords():
15461590

1547-
with pytest.raises(ValueError, match="not yet supported with the new"):
1548-
pq.ParquetDataset("", use_legacy_dataset=False, schema=pa.schema([]))
1549-
15501591
with pytest.raises(ValueError, match="not yet supported with the new"):
15511592
pq.ParquetDataset("", use_legacy_dataset=False, metadata=pa.schema([]))
15521593

@@ -1653,6 +1694,9 @@ def test_parquet_dataset_deprecated_properties(tempdir):
16531694
with pytest.warns(DeprecationWarning, match="'ParquetDataset.fs"):
16541695
dataset.fs
16551696

1697+
with pytest.warns(DeprecationWarning, match="'ParquetDataset.schema'"):
1698+
dataset.schema
1699+
16561700
dataset2 = pq.ParquetDataset(path, use_legacy_dataset=False)
16571701

16581702
with pytest.warns(DeprecationWarning, match="'ParquetDataset.pieces"):

0 commit comments

Comments
 (0)