Skip to content

Commit a4438c5

Browse files
committed
apacheGH-49821: [Python][Parquet] Support per-column keys in direct encryption APIs
Signed-off-by: Yanzhi Xin <[email protected]>
1 parent ca5564e commit a4438c5

5 files changed

Lines changed: 230 additions & 22 deletions

File tree

‎docs/source/python/parquet/parquet_encryption.rst‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -269,10 +269,8 @@ Direct Key Encryption (without KMS)
269269
encryption/decryption properties directly from a plaintext key, bypassing
270270
:class:`~pyarrow.parquet.encryption.CryptoFactory` and the KMS-based flow.
271271

272-
.. note::
273-
Only uniform encryption (a single key for the footer and all columns) is
274-
supported by these functions. For per-column keys, use the
275-
``CryptoFactory``/``EncryptionConfiguration`` flow described above.
272+
Without ``column_keys``, these functions use uniform encryption: the footer
273+
key also encrypts every column.
276274

277275
.. code-block:: python
278276
@@ -288,3 +286,30 @@ encryption/decryption properties directly from a plaintext key, bypassing
288286
... aad_prefix=b'table_id',
289287
... )
290288
>>> pq.read_table('encrypted.parquet', decryption_properties=decryption_props) # doctest: +SKIP
289+
290+
To encrypt selected columns with separate keys, pass ``column_keys`` as a
291+
dictionary mapping full Parquet column paths to raw keys. Unlike
292+
``EncryptionConfiguration.column_keys``, this maps column paths to keys,
293+
rather than master key IDs to lists of column names. Unlisted columns remain
294+
unencrypted. Nested fields use dot-separated leaf paths, such as
295+
``"person.email"``. The dictionary must not be empty.
296+
Specify each nested leaf separately; parent-field aliases cannot be used
297+
for decryption.
298+
299+
.. code-block:: python
300+
301+
>>> import pyarrow as pa
302+
>>> table = pa.table({"sensitive": [1, 2], "public": ["a", "b"]}) # doctest: +SKIP
303+
>>> column_keys = {"sensitive": b'abcdef0123456789'}
304+
>>> props = pe.create_encryption_properties( # doctest: +SKIP
305+
... footer_key=b'0123456789abcdef', column_keys=column_keys)
306+
>>> pq.write_table(table, 'columns.parquet', encryption_properties=props) # doctest: +SKIP
307+
>>> decryption_props = pe.create_decryption_properties( # doctest: +SKIP
308+
... footer_key=b'0123456789abcdef', column_keys=column_keys)
309+
>>> pq.read_table('columns.parquet', decryption_properties=decryption_props) # doctest: +SKIP
310+
311+
Readers only need keys for the encrypted columns they project. The footer key
312+
is still required by these functions, and all supplied keys must be 16, 24,
313+
or 32 bytes. The caller remains responsible for managing and protecting raw
314+
keys; ``CryptoFactory`` provides the higher-level KMS-based key management
315+
flow described above.

‎python/pyarrow/_parquet_encryption.pxd‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121
from pyarrow.includes.common cimport *
2222
from pyarrow.includes.libparquet_encryption cimport *
2323
from pyarrow.includes.libparquet cimport (
24+
c_map,
25+
CColumnDecryptionProperties,
26+
CColumnDecryptionPropertiesBuilder,
27+
CColumnEncryptionProperties,
2428
CSecureString,
2529
CFileDecryptionPropertiesBuilder,
2630
CFileEncryptionPropertiesBuilder,

‎python/pyarrow/_parquet_encryption.pyx‎

Lines changed: 67 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -749,12 +749,29 @@ cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object
749749
raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig))
750750

751751

752+
cdef _validate_column_keys(column_keys):
753+
if not isinstance(column_keys, dict):
754+
raise TypeError("column_keys must be a dict of column paths to bytes")
755+
if not column_keys:
756+
raise ValueError("column_keys must not be empty")
757+
for column_path, key in column_keys.items():
758+
if not isinstance(column_path, str):
759+
raise TypeError("column paths in column_keys must be strings")
760+
if not column_path:
761+
raise ValueError("column paths in column_keys must not be empty")
762+
if not isinstance(key, bytes):
763+
raise TypeError("column keys must be bytes")
764+
if len(key) not in (16, 24, 32):
765+
raise ValueError("column keys must be 16, 24, or 32 bytes")
766+
767+
752768
def create_decryption_properties(
753769
footer_key,
754770
*,
755771
aad_prefix=None,
756772
bint check_footer_integrity=True,
757773
bint allow_plaintext_files=False,
774+
column_keys=None,
758775
):
759776
"""
760777
Create FileDecryptionProperties using a direct footer key.
@@ -769,18 +786,12 @@ def create_decryption_properties(
769786
Parquet key management specification and is interoperable with
770787
other tools and frameworks.
771788
772-
.. note::
773-
Currently only uniform encryption (single key for footer and all
774-
columns) is supported with this method. Per-column keys are not
775-
yet available; files encrypted with per-column keys cannot be
776-
decrypted using this function.
777-
778789
Parameters
779790
----------
780791
footer_key : bytes
781-
The decryption key for the file footer and all columns (uniform
782-
encryption). Must be 16, 24, or 32 bytes for AES-128, AES-192,
783-
or AES-256 respectively.
792+
The decryption key for the file footer and any columns encrypted
793+
with the footer key. Must be 16, 24, or 32 bytes for AES-128,
794+
AES-192, or AES-256 respectively.
784795
aad_prefix : bytes, optional
785796
Additional Authenticated Data prefix. Must match the AAD prefix
786797
that was used during encryption. Required if the AAD prefix was
@@ -791,6 +802,12 @@ def create_decryption_properties(
791802
allow_plaintext_files : bool, default False
792803
Whether to allow reading plaintext (unencrypted) files with
793804
these decryption properties without raising an error.
805+
column_keys : dict[str, bytes], optional
806+
Explicit keys for encrypted columns, mapped by their full Parquet
807+
column paths (dot-separated for nested fields). Keys must be 16, 24,
808+
or 32 bytes. Only columns read need to have keys supplied. Columns
809+
encrypted with the footer key continue to use ``footer_key``.
810+
An empty dictionary is not allowed.
794811
795812
Returns
796813
-------
@@ -814,6 +831,11 @@ def create_decryption_properties(
814831
CSecureString c_footer_key
815832
CFileDecryptionPropertiesBuilder builder
816833
shared_ptr[CFileDecryptionProperties] props
834+
c_map[c_string, shared_ptr[CColumnDecryptionProperties]] c_column_keys
835+
unique_ptr[CColumnDecryptionPropertiesBuilder] column_builder
836+
c_string c_column_path
837+
c_string c_column_key_str
838+
CSecureString c_column_key
817839

818840
if not isinstance(footer_key, bytes):
819841
raise TypeError(
@@ -828,6 +850,18 @@ def create_decryption_properties(
828850
c_footer_key = CSecureString(move(c_footer_key_str))
829851
builder.footer_key(c_footer_key)
830852

853+
if column_keys is not None:
854+
_validate_column_keys(column_keys)
855+
for column_path, key in column_keys.items():
856+
c_column_path = tobytes(column_path)
857+
c_column_key_str = <c_string>key
858+
c_column_key = CSecureString(move(c_column_key_str))
859+
column_builder.reset(
860+
new CColumnDecryptionPropertiesBuilder(c_column_path))
861+
column_builder.get().key(c_column_key)
862+
c_column_keys[c_column_path] = column_builder.get().build()
863+
builder.column_keys(c_column_keys)
864+
831865
if aad_prefix is not None:
832866
if not isinstance(aad_prefix, bytes):
833867
raise TypeError(
@@ -853,6 +887,7 @@ def create_encryption_properties(
853887
bint store_aad_prefix=True,
854888
encryption_algorithm="AES_GCM_V1",
855889
bint plaintext_footer=False,
890+
column_keys=None,
856891
):
857892
"""
858893
Create FileEncryptionProperties using a direct footer key.
@@ -871,18 +906,12 @@ def create_encryption_properties(
871906
prefer it unless you have a specific reason to manage
872907
keys yourself.
873908
874-
.. note::
875-
Currently only uniform encryption (single key for footer and all
876-
columns) is supported with this method. Per-column keys are not
877-
yet available; the provided key encrypts both the footer and
878-
every column.
879-
880909
Parameters
881910
----------
882911
footer_key : bytes
883-
The encryption key for the file footer and all columns (uniform
884-
encryption). Must be 16, 24, or 32 bytes for AES-128, AES-192,
885-
or AES-256 respectively.
912+
The encryption key for the file footer. Also encrypts every column
913+
when ``column_keys`` is not specified (uniform encryption). Must be
914+
16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively.
886915
aad_prefix : bytes, optional
887916
Additional Authenticated Data prefix for cryptographic binding.
888917
store_aad_prefix : bool, default True
@@ -896,6 +925,12 @@ def create_encryption_properties(
896925
plaintext_footer : bool, default False
897926
Whether to leave the file footer unencrypted. When True, file
898927
schema and metadata are readable without a key.
928+
column_keys : dict[str, bytes], optional
929+
Keys for columns to encrypt, mapped by their full Parquet column
930+
paths (dot-separated for nested fields). Keys must be 16, 24, or
931+
32 bytes. When specified, columns not in this dictionary remain
932+
unencrypted. If omitted, all columns use ``footer_key``.
933+
An empty dictionary is not allowed.
899934
900935
Returns
901936
-------
@@ -922,6 +957,10 @@ def create_encryption_properties(
922957
CFileEncryptionPropertiesBuilder* builder
923958
shared_ptr[CFileEncryptionProperties] props
924959
ParquetCipher cipher
960+
c_map[c_string, shared_ptr[CColumnEncryptionProperties]] c_column_keys
961+
c_string c_column_path
962+
c_string c_column_key_str
963+
CSecureString c_column_key
925964

926965
if not isinstance(footer_key, bytes):
927966
raise TypeError(
@@ -939,6 +978,16 @@ def create_encryption_properties(
939978
try:
940979
builder.algorithm(cipher)
941980

981+
if column_keys is not None:
982+
_validate_column_keys(column_keys)
983+
for column_path, key in column_keys.items():
984+
c_column_path = tobytes(column_path)
985+
c_column_key_str = <c_string>key
986+
c_column_key = CSecureString(move(c_column_key_str))
987+
c_column_keys[c_column_path] = \
988+
CColumnEncryptionProperties.WithColumnKey(c_column_key, b"")
989+
builder.encrypted_columns(c_column_keys)
990+
942991
if aad_prefix is not None:
943992
if not isinstance(aad_prefix, bytes):
944993
raise TypeError(

‎python/pyarrow/includes/libparquet.pxd‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
# distutils: language = c++
1919

20+
from libcpp.map cimport map as c_map
21+
2022
from pyarrow.includes.common cimport *
2123
from pyarrow.includes.libarrow cimport (Type, CChunkedArray, CScalar, CSchema,
2224
CStatus, CTable, CMemoryPool, CBuffer,
@@ -638,6 +640,22 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
638640
const COutputStream* sink)
639641

640642
cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil:
643+
cdef cppclass CColumnEncryptionProperties\
644+
" parquet::ColumnEncryptionProperties":
645+
@staticmethod
646+
shared_ptr[CColumnEncryptionProperties] WithColumnKey(
647+
CSecureString key, c_string key_metadata) except +
648+
649+
cdef cppclass CColumnDecryptionProperties\
650+
" parquet::ColumnDecryptionProperties":
651+
pass
652+
653+
cdef cppclass CColumnDecryptionPropertiesBuilder\
654+
" parquet::ColumnDecryptionProperties::Builder":
655+
CColumnDecryptionPropertiesBuilder(c_string name) except +
656+
CColumnDecryptionPropertiesBuilder* key(CSecureString key) except +
657+
shared_ptr[CColumnDecryptionProperties] build() except +
658+
641659
cdef cppclass CFileDecryptionProperties\
642660
" parquet::FileDecryptionProperties":
643661
pass
@@ -647,6 +665,8 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil:
647665
CFileDecryptionPropertiesBuilder() except +
648666
CFileDecryptionPropertiesBuilder* footer_key(
649667
CSecureString footer_key) except +
668+
CFileDecryptionPropertiesBuilder* column_keys(
669+
c_map[c_string, shared_ptr[CColumnDecryptionProperties]] column_keys) except +
650670
CFileDecryptionPropertiesBuilder* aad_prefix(
651671
c_string aad_prefix) except +
652672
CFileDecryptionPropertiesBuilder* disable_footer_signature_verification() except +
@@ -666,4 +686,6 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil:
666686
CFileEncryptionPropertiesBuilder* aad_prefix(
667687
c_string aad_prefix) except +
668688
CFileEncryptionPropertiesBuilder* disable_aad_prefix_storage() except +
689+
CFileEncryptionPropertiesBuilder* encrypted_columns(
690+
c_map[c_string, shared_ptr[CColumnEncryptionProperties]] columns) except +
669691
shared_ptr[CFileEncryptionProperties] build() except +

‎python/pyarrow/tests/parquet/test_encryption.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,114 @@ def test_roundtrip_key_sizes(self, tempdir, data_table, key):
880880
result = pq.read_table(path, decryption_properties=dec_props)
881881
assert data_table.equals(result)
882882

883+
@pytest.mark.parametrize("plaintext_footer", [False, True])
884+
@pytest.mark.parametrize("store_aad_prefix", [False, True])
885+
def test_per_column_keys_roundtrip(
886+
self, tempdir, data_table, plaintext_footer, store_aad_prefix
887+
):
888+
path = tempdir / "direct_columns.parquet"
889+
column_keys = {"a": DIRECT_KEY_192, "b": DIRECT_KEY_256}
890+
enc_props = pe.create_encryption_properties(
891+
DIRECT_KEY_128,
892+
column_keys=column_keys,
893+
plaintext_footer=plaintext_footer,
894+
aad_prefix=DIRECT_AAD_PREFIX,
895+
store_aad_prefix=store_aad_prefix,
896+
)
897+
pq.write_table(data_table, path, encryption_properties=enc_props)
898+
899+
dec_props = pe.create_decryption_properties(
900+
DIRECT_KEY_128,
901+
column_keys=column_keys,
902+
aad_prefix=DIRECT_AAD_PREFIX,
903+
)
904+
result = pq.read_table(path, decryption_properties=dec_props)
905+
assert result.equals(data_table)
906+
907+
def test_per_column_keys_projection(self, tempdir, data_table):
908+
path = tempdir / "direct_projection.parquet"
909+
enc_props = pe.create_encryption_properties(
910+
DIRECT_KEY_128,
911+
column_keys={"a": DIRECT_KEY_192, "b": DIRECT_KEY_256},
912+
)
913+
pq.write_table(data_table, path, encryption_properties=enc_props)
914+
915+
# A key is only required for an encrypted column being read.
916+
result = pq.read_table(
917+
path, columns=["c"],
918+
decryption_properties=pe.create_decryption_properties(DIRECT_KEY_128),
919+
)
920+
assert result.equals(data_table.select(["c"]))
921+
result = pq.read_table(
922+
path, columns=["a", "c"],
923+
decryption_properties=pe.create_decryption_properties(
924+
DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_192}),
925+
)
926+
assert result.equals(data_table.select(["a", "c"]))
927+
928+
with pytest.raises(IOError, match="key"):
929+
pq.read_table(
930+
path,
931+
decryption_properties=pe.create_decryption_properties(
932+
DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_192}),
933+
)
934+
935+
def test_per_column_wrong_key(self, tempdir, data_table):
936+
path = tempdir / "direct_wrong_column_key.parquet"
937+
enc_props = pe.create_encryption_properties(
938+
DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_192})
939+
pq.write_table(data_table, path, encryption_properties=enc_props)
940+
941+
with pytest.raises(IOError, match="decrypt"):
942+
pq.read_table(
943+
path,
944+
decryption_properties=pe.create_decryption_properties(
945+
DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_256}),
946+
)
947+
948+
def test_per_column_nested_path(self, tempdir):
949+
table = pa.table({"details": [{"private": 1, "public": "a"},
950+
{"private": 2, "public": "b"}]})
951+
path = tempdir / "direct_nested.parquet"
952+
column_keys = {"details.private": DIRECT_KEY_192}
953+
pq.write_table(
954+
table, path,
955+
encryption_properties=pe.create_encryption_properties(
956+
DIRECT_KEY_128, column_keys=column_keys),
957+
)
958+
result = pq.read_table(
959+
path,
960+
decryption_properties=pe.create_decryption_properties(
961+
DIRECT_KEY_128, column_keys=column_keys),
962+
)
963+
assert result.equals(table)
964+
965+
def test_per_column_unknown_path(self, tempdir, data_table):
966+
enc_props = pe.create_encryption_properties(
967+
DIRECT_KEY_128, column_keys={"missing": DIRECT_KEY_192})
968+
with pytest.raises(
969+
IOError, match="Encrypted column missing not in file schema"
970+
):
971+
pq.write_table(
972+
data_table, tempdir / "direct_unknown_column.parquet",
973+
encryption_properties=enc_props,
974+
)
975+
976+
@pytest.mark.parametrize("factory", [
977+
"create_encryption_properties", "create_decryption_properties",
978+
])
979+
@pytest.mark.parametrize("column_keys, error, message", [
980+
([], TypeError, "column_keys must be a dict"),
981+
({}, ValueError, "column_keys must not be empty"),
982+
({1: DIRECT_KEY_128}, TypeError, "column paths.*must be strings"),
983+
({"": DIRECT_KEY_128}, ValueError, "column paths.*must not be empty"),
984+
({"a": "0123456789abcdef"}, TypeError, "column keys must be bytes"),
985+
({"a": b"short"}, ValueError, "column keys must be 16, 24, or 32 bytes"),
986+
])
987+
def test_invalid_column_keys(self, factory, column_keys, error, message):
988+
with pytest.raises(error, match=message):
989+
getattr(pe, factory)(DIRECT_KEY_128, column_keys=column_keys)
990+
883991
def test_roundtrip_with_aad_prefix(self, tempdir, data_table):
884992
path = tempdir / "direct_aad.parquet"
885993

0 commit comments

Comments
 (0)