Skip to content

Commit 88bccab

Browse files
andersonm-ibmggershinsky
authored andcommitted
ARROW-14114: [C++][Parquet] Fix multi-threaded read of PME files
Change AesDecryptor to be per Decryptor, instead of shared. This solves the problem of reading with PME using multiple threads. Details: It was discovered when exposing high-level PME in PyArrow that reading an encrypted parquet file in PyArrow intermittently fails decryption finalization and sometime fails with Segmentation fault. The same in C++ reading an encrypted parquet with FileReader.ReadTable() multithreaded (with set_use_threads(true) ). The current implementation uses two caches: meta_decryptor_ and data_decryptor_ , for AesDecryptors, and every Decryptor gets the same AesDecryptor with AesDecryptorImpl from this cache. However, AesDecryptor::AesDecryptorImpl::GcmDecrypt() and AesDecryptor::AesDecryptorImpl::CtrDecrypt() use ctx_ member of type EVP_CIPHER_CTX from OpenSSL, which shouldn't be used from multiple threads concurrently. So, instead of sharing the same AesDecryptor between all Decryptors, an AesDecryptor will be created per Decryptor, which is per column. Co-authored-by: Gidon Gershinsky <[email protected]> CC @thamht4190 @pitrou @revit13 Closes #12778 from andersonm-ibm/multithreaded_read Lead-authored-by: Maya Anderson <[email protected]> Co-authored-by: Gidon Gershinsky <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent 860ce72 commit 88bccab

6 files changed

Lines changed: 40 additions & 61 deletions

File tree

‎cpp/src/parquet/encryption/encryption_internal.cc‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -427,15 +427,16 @@ AesDecryptor::AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadat
427427
: impl_{std::unique_ptr<AesDecryptorImpl>(
428428
new AesDecryptorImpl(alg_id, key_len, metadata, contains_length))} {}
429429

430-
AesDecryptor* AesDecryptor::Make(ParquetCipher::type alg_id, int key_len, bool metadata,
431-
std::vector<AesDecryptor*>* all_decryptors) {
430+
std::shared_ptr<AesDecryptor> AesDecryptor::Make(
431+
ParquetCipher::type alg_id, int key_len, bool metadata,
432+
std::vector<std::weak_ptr<AesDecryptor>>* all_decryptors) {
432433
if (ParquetCipher::AES_GCM_V1 != alg_id && ParquetCipher::AES_GCM_CTR_V1 != alg_id) {
433434
std::stringstream ss;
434435
ss << "Crypto algorithm " << alg_id << " is not supported";
435436
throw ParquetException(ss.str());
436437
}
437438

438-
AesDecryptor* decryptor = new AesDecryptor(alg_id, key_len, metadata);
439+
auto decryptor = std::make_shared<AesDecryptor>(alg_id, key_len, metadata);
439440
if (all_decryptors != nullptr) {
440441
all_decryptors->push_back(decryptor);
441442
}

‎cpp/src/parquet/encryption/encryption_internal.h‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,17 @@ class AesDecryptor {
8888
explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata,
8989
bool contains_length = true);
9090

91-
static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
92-
std::vector<AesDecryptor*>* all_decryptors);
91+
/// \brief Factory function to create an AesDecryptor
92+
///
93+
/// \param alg_id the encryption algorithm to use
94+
/// \param key_len key length. Possible values: 16, 24, 32 bytes.
95+
/// \param metadata if true then this is a metadata decryptor
96+
/// \param all_decryptors A weak reference to all decryptors that need to be wiped
97+
/// out when decryption is finished
98+
/// \return shared pointer to a new AesDecryptor
99+
static std::shared_ptr<AesDecryptor> Make(
100+
ParquetCipher::type alg_id, int key_len, bool metadata,
101+
std::vector<std::weak_ptr<AesDecryptor>>* all_decryptors);
93102

94103
~AesDecryptor();
95104
void WipeOut();

‎cpp/src/parquet/encryption/encryption_internal_nossl.cc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,9 @@ AesDecryptor::AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadat
8686
ThrowOpenSSLRequiredException();
8787
}
8888

89-
AesDecryptor* AesDecryptor::Make(ParquetCipher::type alg_id, int key_len, bool metadata,
90-
std::vector<AesDecryptor*>* all_decryptors) {
89+
std::shared_ptr<AesDecryptor> AesDecryptor::Make(
90+
ParquetCipher::type alg_id, int key_len, bool metadata,
91+
std::vector<std::weak_ptr<AesDecryptor>>* all_decryptors) {
9192
return NULLPTR;
9293
}
9394

‎cpp/src/parquet/encryption/internal_file_decryptor.cc‎

Lines changed: 16 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222
namespace parquet {
2323

2424
// Decryptor
25-
Decryptor::Decryptor(encryption::AesDecryptor* aes_decryptor, const std::string& key,
26-
const std::string& file_aad, const std::string& aad,
27-
::arrow::MemoryPool* pool)
25+
Decryptor::Decryptor(std::shared_ptr<encryption::AesDecryptor> aes_decryptor,
26+
const std::string& key, const std::string& file_aad,
27+
const std::string& aad, ::arrow::MemoryPool* pool)
2828
: aes_decryptor_(aes_decryptor),
2929
key_(key),
3030
file_aad_(file_aad),
@@ -61,7 +61,9 @@ InternalFileDecryptor::InternalFileDecryptor(FileDecryptionProperties* propertie
6161
void InternalFileDecryptor::WipeOutDecryptionKeys() {
6262
properties_->WipeOutDecryptionKeys();
6363
for (auto const& i : all_decryptors_) {
64-
i->WipeOut();
64+
if (auto aes_decryptor = i.lock()) {
65+
aes_decryptor->WipeOut();
66+
}
6567
}
6668
}
6769

@@ -134,8 +136,11 @@ std::shared_ptr<Decryptor> InternalFileDecryptor::GetFooterDecryptor(
134136

135137
// Create both data and metadata decryptors to avoid redundant retrieval of key
136138
// from the key_retriever.
137-
auto aes_metadata_decryptor = GetMetaAesDecryptor(footer_key.size());
138-
auto aes_data_decryptor = GetDataAesDecryptor(footer_key.size());
139+
int key_len = static_cast<int>(footer_key.size());
140+
auto aes_metadata_decryptor = encryption::AesDecryptor::Make(
141+
algorithm_, key_len, /*metadata=*/true, &all_decryptors_);
142+
auto aes_data_decryptor = encryption::AesDecryptor::Make(
143+
algorithm_, key_len, /*metadata=*/false, &all_decryptors_);
139144

140145
footer_metadata_decryptor_ = std::make_shared<Decryptor>(
141146
aes_metadata_decryptor, footer_key, file_aad_, aad, pool_);
@@ -195,8 +200,11 @@ std::shared_ptr<Decryptor> InternalFileDecryptor::GetColumnDecryptor(
195200

196201
// Create both data and metadata decryptors to avoid redundant retrieval of key
197202
// using the key_retriever.
198-
auto aes_metadata_decryptor = GetMetaAesDecryptor(column_key.size());
199-
auto aes_data_decryptor = GetDataAesDecryptor(column_key.size());
203+
int key_len = static_cast<int>(column_key.size());
204+
auto aes_metadata_decryptor = encryption::AesDecryptor::Make(
205+
algorithm_, key_len, /*metadata=*/true, &all_decryptors_);
206+
auto aes_data_decryptor = encryption::AesDecryptor::Make(
207+
algorithm_, key_len, /*metadata=*/false, &all_decryptors_);
200208

201209
column_metadata_map_[column_path] = std::make_shared<Decryptor>(
202210
aes_metadata_decryptor, column_key, file_aad_, aad, pool_);
@@ -207,34 +215,4 @@ std::shared_ptr<Decryptor> InternalFileDecryptor::GetColumnDecryptor(
207215
return column_data_map_[column_path];
208216
}
209217

210-
int InternalFileDecryptor::MapKeyLenToDecryptorArrayIndex(int key_len) {
211-
if (key_len == 16)
212-
return 0;
213-
else if (key_len == 24)
214-
return 1;
215-
else if (key_len == 32)
216-
return 2;
217-
throw ParquetException("decryption key must be 16, 24 or 32 bytes in length");
218-
}
219-
220-
encryption::AesDecryptor* InternalFileDecryptor::GetMetaAesDecryptor(size_t key_size) {
221-
int key_len = static_cast<int>(key_size);
222-
int index = MapKeyLenToDecryptorArrayIndex(key_len);
223-
if (meta_decryptor_[index] == nullptr) {
224-
meta_decryptor_[index].reset(
225-
encryption::AesDecryptor::Make(algorithm_, key_len, true, &all_decryptors_));
226-
}
227-
return meta_decryptor_[index].get();
228-
}
229-
230-
encryption::AesDecryptor* InternalFileDecryptor::GetDataAesDecryptor(size_t key_size) {
231-
int key_len = static_cast<int>(key_size);
232-
int index = MapKeyLenToDecryptorArrayIndex(key_len);
233-
if (data_decryptor_[index] == nullptr) {
234-
data_decryptor_[index].reset(
235-
encryption::AesDecryptor::Make(algorithm_, key_len, false, &all_decryptors_));
236-
}
237-
return data_decryptor_[index].get();
238-
}
239-
240218
} // namespace parquet

‎cpp/src/parquet/encryption/internal_file_decryptor.h‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class FileDecryptionProperties;
3535

3636
class PARQUET_EXPORT Decryptor {
3737
public:
38-
Decryptor(encryption::AesDecryptor* decryptor, const std::string& key,
38+
Decryptor(std::shared_ptr<encryption::AesDecryptor> decryptor, const std::string& key,
3939
const std::string& file_aad, const std::string& aad,
4040
::arrow::MemoryPool* pool);
4141

@@ -47,7 +47,7 @@ class PARQUET_EXPORT Decryptor {
4747
int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
4848

4949
private:
50-
encryption::AesDecryptor* aes_decryptor_;
50+
std::shared_ptr<encryption::AesDecryptor> aes_decryptor_;
5151
std::string key_;
5252
std::string file_aad_;
5353
std::string aad_;
@@ -97,12 +97,9 @@ class InternalFileDecryptor {
9797
std::shared_ptr<Decryptor> footer_data_decryptor_;
9898
ParquetCipher::type algorithm_;
9999
std::string footer_key_metadata_;
100-
std::vector<encryption::AesDecryptor*> all_decryptors_;
101-
102-
/// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three
103-
// types of meta_decryptors and data_decryptors.
104-
std::unique_ptr<encryption::AesDecryptor> meta_decryptor_[3];
105-
std::unique_ptr<encryption::AesDecryptor> data_decryptor_[3];
100+
// A weak reference to all decryptors that need to be wiped out when decryption is
101+
// finished
102+
std::vector<std::weak_ptr<encryption::AesDecryptor>> all_decryptors_;
106103

107104
::arrow::MemoryPool* pool_;
108105

@@ -111,11 +108,6 @@ class InternalFileDecryptor {
111108
const std::string& column_key_metadata,
112109
const std::string& aad,
113110
bool metadata = false);
114-
115-
encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size);
116-
encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size);
117-
118-
int MapKeyLenToDecryptorArrayIndex(int key_len);
119111
};
120112

121113
} // namespace parquet

‎python/pyarrow/tests/parquet/test_encryption.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ def read_encrypted_parquet(path, decryption_config,
126126

127127
result = pq.ParquetFile(
128128
path, decryption_properties=file_decryption_properties)
129-
return result.read(use_threads=False)
129+
return result.read(use_threads=True)
130130

131131

132132
def test_encrypted_parquet_write_read_wrong_key(tempdir, data_table):
@@ -486,8 +486,6 @@ def kms_factory(kms_connection_configuration):
486486
kms_connection_config, crypto_factory)
487487

488488

489-
@pytest.mark.skip(reason="ARROW-14114: Multithreaded read sometimes fails"
490-
"decryption finalization or with Segmentation fault")
491489
def test_encrypted_parquet_loop(tempdir, data_table, basic_encryption_config):
492490
"""Write an encrypted parquet, verify it's encrypted,
493491
and then read it multithreaded in a loop."""

0 commit comments

Comments
 (0)