Skip to content

Commit f78c7ac

Browse files
smaheshwar-pltrMottl
authored andcommitted
apacheGH-47435: [Python][Parquet] Add direct key encryption/decryption API (apache#49667)
### Rationale for this change See apache#47435. ### What changes are included in this PR? Adds direct encryption / decryption Python API ### Are these changes tested? Yes, see PR. ### Are there any user-facing changes? Yes, new Python bindings. * GitHub Issue: apache#47435 Authored-by: Sreesh Maheshwar <[email protected]> Signed-off-by: Adam Reeve <[email protected]>
1 parent 274544b commit f78c7ac

6 files changed

Lines changed: 469 additions & 2 deletions

File tree

‎docs/source/python/api/formats.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ Encrypted Parquet Files
119119
KmsConnectionConfig
120120
EncryptionConfiguration
121121
DecryptionConfiguration
122+
create_encryption_properties
123+
create_decryption_properties
122124

123125
.. _api.orc:
124126

‎python/pyarrow/_parquet_encryption.pxd‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@
2020

2121
from pyarrow.includes.common cimport *
2222
from pyarrow.includes.libparquet_encryption cimport *
23+
from pyarrow.includes.libparquet cimport (
24+
CSecureString,
25+
CFileDecryptionPropertiesBuilder,
26+
CFileEncryptionPropertiesBuilder,
27+
)
2328
from pyarrow._parquet cimport (ParquetCipher,
2429
CFileEncryptionProperties,
2530
CFileDecryptionProperties,

‎python/pyarrow/_parquet_encryption.pyx‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -711,3 +711,212 @@ cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object
711711
if isinstance(decryptionconfig, DecryptionConfiguration):
712712
return (<DecryptionConfiguration> decryptionconfig).unwrap()
713713
raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig))
714+
715+
716+
def create_decryption_properties(
717+
footer_key,
718+
*,
719+
aad_prefix=None,
720+
bint check_footer_integrity=True,
721+
bint allow_plaintext_files=False,
722+
):
723+
"""
724+
Create FileDecryptionProperties using a direct footer key.
725+
726+
This is a low-level API that constructs decryption properties directly
727+
from a plaintext key, bypassing the KMS-based :class:`CryptoFactory`.
728+
It is intended for callers that manage key wrapping and storage
729+
themselves (e.g. an application-level scheme).
730+
731+
For most use cases, prefer the higher-level :class:`CryptoFactory`
732+
with :class:`DecryptionConfiguration`, which implements the full
733+
Parquet key management specification and is interoperable with
734+
other tools and frameworks.
735+
736+
.. note::
737+
Currently only uniform encryption (single key for footer and all
738+
columns) is supported with this method. Per-column keys are not
739+
yet available; files encrypted with per-column keys cannot be
740+
decrypted using this function.
741+
742+
Parameters
743+
----------
744+
footer_key : bytes
745+
The decryption key for the file footer and all columns (uniform
746+
encryption). Must be 16, 24, or 32 bytes for AES-128, AES-192,
747+
or AES-256 respectively.
748+
aad_prefix : bytes, optional
749+
Additional Authenticated Data prefix. Must match the AAD prefix
750+
that was used during encryption. Required if the AAD prefix was
751+
not stored in the file metadata during encryption.
752+
check_footer_integrity : bool, default True
753+
Whether to verify footer integrity using the signature stored
754+
in the file. Set to False only for debugging.
755+
allow_plaintext_files : bool, default False
756+
Whether to allow reading plaintext (unencrypted) files with
757+
these decryption properties without raising an error.
758+
759+
Returns
760+
-------
761+
FileDecryptionProperties
762+
Properties that can be passed to :func:`~pyarrow.parquet.read_table`,
763+
:class:`~pyarrow.parquet.ParquetFile`, or
764+
:class:`~pyarrow.dataset.ParquetFragmentScanOptions`.
765+
766+
Examples
767+
--------
768+
>>> import pyarrow.parquet as pq
769+
>>> import pyarrow.parquet.encryption as pe
770+
>>> props = pe.create_decryption_properties(
771+
... footer_key=b'0123456789abcdef',
772+
... aad_prefix=b'table_id',
773+
... )
774+
>>> table = pq.read_table('encrypted.parquet', decryption_properties=props)
775+
"""
776+
cdef:
777+
c_string c_footer_key_str
778+
CSecureString c_footer_key
779+
CFileDecryptionPropertiesBuilder builder
780+
shared_ptr[CFileDecryptionProperties] props
781+
782+
if not isinstance(footer_key, bytes):
783+
raise TypeError(
784+
f"footer_key must be bytes, not {type(footer_key).__name__}"
785+
)
786+
if len(footer_key) not in (16, 24, 32):
787+
raise ValueError(
788+
f"footer_key must be 16, 24, or 32 bytes, got {len(footer_key)}"
789+
)
790+
791+
c_footer_key_str = <c_string>footer_key
792+
c_footer_key = CSecureString(move(c_footer_key_str))
793+
builder.footer_key(c_footer_key)
794+
795+
if aad_prefix is not None:
796+
if not isinstance(aad_prefix, bytes):
797+
raise TypeError(
798+
f"aad_prefix must be bytes, not {type(aad_prefix).__name__}"
799+
)
800+
builder.aad_prefix(<c_string>aad_prefix)
801+
802+
if not check_footer_integrity:
803+
builder.disable_footer_signature_verification()
804+
805+
if allow_plaintext_files:
806+
builder.plaintext_files_allowed()
807+
808+
props = builder.build()
809+
810+
return FileDecryptionProperties.wrap(props)
811+
812+
813+
def create_encryption_properties(
814+
footer_key,
815+
*,
816+
aad_prefix=None,
817+
bint store_aad_prefix=True,
818+
encryption_algorithm="AES_GCM_V1",
819+
bint plaintext_footer=False,
820+
):
821+
"""
822+
Create FileEncryptionProperties using a direct footer key.
823+
824+
This is a low-level API that constructs encryption properties directly
825+
from a plaintext key, bypassing the KMS-based :class:`CryptoFactory`.
826+
It is intended for callers that manage key wrapping and storage
827+
themselves (e.g. an application-level scheme).
828+
829+
.. warning::
830+
The caller is responsible for key management best practices.
831+
Reusing the same key for multiple files without unique data keys
832+
weakens AES-GCM security. The higher-level :class:`CryptoFactory`
833+
with :class:`EncryptionConfiguration` handles this automatically
834+
and is interoperable with other tools and frameworks --
835+
prefer it unless you have a specific reason to manage
836+
keys yourself.
837+
838+
.. note::
839+
Currently only uniform encryption (single key for footer and all
840+
columns) is supported with this method. Per-column keys are not
841+
yet available; the provided key encrypts both the footer and
842+
every column.
843+
844+
Parameters
845+
----------
846+
footer_key : bytes
847+
The encryption key for the file footer and all columns (uniform
848+
encryption). Must be 16, 24, or 32 bytes for AES-128, AES-192,
849+
or AES-256 respectively.
850+
aad_prefix : bytes, optional
851+
Additional Authenticated Data prefix for cryptographic binding.
852+
store_aad_prefix : bool, default True
853+
Whether to store the AAD prefix in the Parquet file metadata.
854+
Set to False when the AAD prefix will be supplied externally
855+
at read time.
856+
Only meaningful when *aad_prefix* is provided.
857+
encryption_algorithm : str, default "AES_GCM_V1"
858+
Encryption algorithm. Either ``"AES_GCM_V1"`` or
859+
``"AES_GCM_CTR_V1"``.
860+
plaintext_footer : bool, default False
861+
Whether to leave the file footer unencrypted. When True, file
862+
schema and metadata are readable without a key.
863+
864+
Returns
865+
-------
866+
FileEncryptionProperties
867+
Properties that can be passed to :func:`~pyarrow.parquet.write_table` or
868+
:class:`~pyarrow.parquet.ParquetWriter`.
869+
870+
Examples
871+
--------
872+
>>> import pyarrow as pa
873+
>>> import pyarrow.parquet as pq
874+
>>> import pyarrow.parquet.encryption as pe
875+
>>> table = pa.table({'col': [1, 2, 3]})
876+
>>> props = pe.create_encryption_properties(
877+
... footer_key=b'0123456789abcdef',
878+
... aad_prefix=b'table_id',
879+
... store_aad_prefix=False,
880+
... )
881+
>>> pq.write_table(table, 'encrypted.parquet', encryption_properties=props)
882+
"""
883+
cdef:
884+
c_string c_footer_key_str
885+
CSecureString c_footer_key
886+
CFileEncryptionPropertiesBuilder* builder
887+
shared_ptr[CFileEncryptionProperties] props
888+
ParquetCipher cipher
889+
890+
if not isinstance(footer_key, bytes):
891+
raise TypeError(
892+
f"footer_key must be bytes, not {type(footer_key).__name__}"
893+
)
894+
if len(footer_key) not in (16, 24, 32):
895+
raise ValueError(
896+
f"footer_key must be 16, 24, or 32 bytes, got {len(footer_key)}"
897+
)
898+
899+
cipher = cipher_from_name(encryption_algorithm)
900+
c_footer_key_str = <c_string>footer_key
901+
c_footer_key = CSecureString(move(c_footer_key_str))
902+
builder = new CFileEncryptionPropertiesBuilder(c_footer_key)
903+
try:
904+
builder.algorithm(cipher)
905+
906+
if aad_prefix is not None:
907+
if not isinstance(aad_prefix, bytes):
908+
raise TypeError(
909+
f"aad_prefix must be bytes, not {type(aad_prefix).__name__}"
910+
)
911+
builder.aad_prefix(<c_string>aad_prefix)
912+
if not store_aad_prefix:
913+
builder.disable_aad_prefix_storage()
914+
915+
if plaintext_footer:
916+
builder.set_plaintext_footer()
917+
918+
props = builder.build()
919+
finally:
920+
del builder
921+
922+
return FileEncryptionProperties.wrap(props)

‎python/pyarrow/includes/libparquet.pxd‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ from pyarrow.includes.libarrow cimport (Type, CChunkedArray, CScalar, CSchema,
2222
CStatus, CTable, CMemoryPool, CBuffer,
2323
CKeyValueMetadata, CRandomAccessFile,
2424
COutputStream, CCacheOptions,
25-
TimeUnit, CRecordBatchReader)
25+
TimeUnit, CRecordBatchReader,
26+
CSecureString)
2627

2728

2829
cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil:
@@ -635,6 +636,28 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil:
635636
" parquet::FileDecryptionProperties":
636637
pass
637638

639+
cdef cppclass CFileDecryptionPropertiesBuilder\
640+
" parquet::FileDecryptionProperties::Builder":
641+
CFileDecryptionPropertiesBuilder() except +
642+
CFileDecryptionPropertiesBuilder* footer_key(
643+
CSecureString footer_key) except +
644+
CFileDecryptionPropertiesBuilder* aad_prefix(
645+
c_string aad_prefix) except +
646+
CFileDecryptionPropertiesBuilder* disable_footer_signature_verification() except +
647+
CFileDecryptionPropertiesBuilder* plaintext_files_allowed() except +
648+
shared_ptr[CFileDecryptionProperties] build() except +
649+
638650
cdef cppclass CFileEncryptionProperties\
639651
" parquet::FileEncryptionProperties":
640652
pass
653+
654+
cdef cppclass CFileEncryptionPropertiesBuilder\
655+
" parquet::FileEncryptionProperties::Builder":
656+
CFileEncryptionPropertiesBuilder(CSecureString footer_key) except +
657+
CFileEncryptionPropertiesBuilder* set_plaintext_footer() except +
658+
CFileEncryptionPropertiesBuilder* algorithm(
659+
ParquetCipher parquet_cipher) except +
660+
CFileEncryptionPropertiesBuilder* aad_prefix(
661+
c_string aad_prefix) except +
662+
CFileEncryptionPropertiesBuilder* disable_aad_prefix_storage() except +
663+
shared_ptr[CFileEncryptionProperties] build() except +

‎python/pyarrow/parquet/encryption.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,6 @@
2020
EncryptionConfiguration,
2121
DecryptionConfiguration,
2222
KmsConnectionConfig,
23-
KmsClient)
23+
KmsClient,
24+
create_encryption_properties,
25+
create_decryption_properties)

0 commit comments

Comments
 (0)