Skip to content

Commit ba754c4

Browse files
autoCAT6raulcd
authored andcommitted
GH-51669: [C++][Parquet] Preserve cipher in plaintext footer metadata (#54884)
### Rationale for this change Closes #51669. CTR files with plaintext footers incorrectly declare GCM, causing encrypted-column reads to fail. ### What changes are included in this PR? * Serialize the configured file encryption algorithm directly. * Select AES-GCM explicitly for plaintext footer signing and verification. * Add generated GCM/CTR, footer and AAD cases, including an on-disk GCM tag check independent of reader selection. Original report by [dotnwat](https://github.com/dotnwat) in #51669. The follow-up incorporates [pitrou's review](#54884 (comment)) and [adamreeve's clarification](#54884 (comment)). ### Are these changes tested? * Encryption: 45 passed; metadata: 50 passed; writer: 296 passed, with 2 existing disabled tests. * Released parquet-java 1.18.1 interoperability: 64 read outcomes and 8 raw metadata inspections matched their controls. * clang-format 18 and `git diff --check` pass. ### Are there any user-facing changes? New CTR/plaintext-footer files declare the correct algorithm. No public API changes. **This PR contains a "Critical Fix"**: it corrects invalid encryption algorithm metadata emitted by the writer. ### Was AI used for this PR? AI was used to generate the implementation, regression tests, and PR description. **PR code and description written by:** * [ ] Human * [x] AI **Reviewed before submission by:** * [x] Human * [x] AI * [ ] Not reviewed * GitHub Issue: #51669 Authored-by: Yanzhi Xin <[email protected]> Signed-off-by: Adam Reeve <[email protected]>
1 parent 16f1823 commit ba754c4

3 files changed

Lines changed: 186 additions & 16 deletions

File tree

‎cpp/src/parquet/encryption/internal_file_encryptor.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,9 @@ std::shared_ptr<Encryptor> InternalFileEncryptor::GetFooterSigningEncryptor() {
6868
return footer_signing_encryptor_;
6969
}
7070

71-
ParquetCipher::type algorithm = properties_->algorithm().algorithm;
71+
// A plaintext footer is always signed with AES-GCM, regardless of the algorithm
72+
// configured for encryption.
73+
const auto algorithm = ParquetCipher::AES_GCM_V1;
7274
std::string footer_aad = encryption::CreateFooterAad(properties_->file_aad());
7375
const SecureString& footer_signing_key = properties_->footer_key();
7476
auto aes_encryptor = GetMetaAesEncryptor(algorithm, footer_signing_key.size());

‎cpp/src/parquet/encryption/write_configurations_test.cc‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,18 @@
1919

2020
#include <stdio.h>
2121

22+
#include <stdexcept>
23+
#include <tuple>
24+
#include <vector>
25+
2226
#include <arrow/io/file.h>
2327

28+
#include "arrow/util/endian.h"
29+
#include "arrow/util/ubsan.h"
30+
2431
#include "parquet/column_reader.h"
2532
#include "parquet/column_writer.h"
33+
#include "parquet/encryption/encryption_internal.h"
2634
#include "parquet/encryption/test_encryption_util.h"
2735
#include "parquet/file_reader.h"
2836
#include "parquet/file_writer.h"
@@ -195,6 +203,170 @@ TEST_F(TestEncryptionConfiguration, EncryptTwoColumnsAndFooterUseAES_GCM_CTR) {
195203
"tmp_encrypt_columns_and_footer_ctr.parquet.encrypted"));
196204
}
197205

206+
enum class AadPrefixMode { None, Stored, External };
207+
208+
class TestGeneratedEncryptionConfiguration
209+
: public ::testing::TestWithParam<
210+
std::tuple<ParquetCipher::type, bool, AadPrefixMode>> {
211+
public:
212+
void SetUp() override {
213+
PARQUET_ASSIGN_OR_THROW(temp_dir_, temp_data_dir());
214+
file_path_ = temp_dir_->path().ToString() + "algorithm_metadata.parquet.encrypted";
215+
216+
std::map<std::string, std::shared_ptr<ColumnEncryptionProperties>> columns;
217+
columns[kDoubleFieldName] =
218+
ColumnEncryptionProperties::WithColumnKey(kColumnEncryptionKey1, "kc1");
219+
columns[kFloatFieldName] =
220+
ColumnEncryptionProperties::WithColumnKey(kColumnEncryptionKey2, "kc2");
221+
columns[kBooleanFieldName] = ColumnEncryptionProperties::Unencrypted();
222+
223+
FileEncryptionProperties::Builder builder(kFooterEncryptionKey);
224+
builder.footer_key_metadata("kf")
225+
->encrypted_columns(std::move(columns))
226+
->algorithm(std::get<0>(GetParam()));
227+
if (std::get<1>(GetParam())) {
228+
builder.set_plaintext_footer();
229+
}
230+
if (std::get<2>(GetParam()) != AadPrefixMode::None) {
231+
builder.aad_prefix(kFileName);
232+
}
233+
if (std::get<2>(GetParam()) == AadPrefixMode::External) {
234+
builder.disable_aad_prefix_storage();
235+
}
236+
auto properties = builder.build();
237+
file_aad_ = properties->file_aad();
238+
FileEncryptor().EncryptFile(file_path_, std::move(properties));
239+
}
240+
241+
protected:
242+
enum class Keys { All, MissingColumn, WrongFooter };
243+
enum class AadPrefix { Match, Missing, Wrong };
244+
245+
std::shared_ptr<FileDecryptionProperties> MakeDecryptionProperties(
246+
Keys keys = Keys::All, AadPrefix prefix = AadPrefix::Match) {
247+
auto retriever = std::make_shared<StringKeyIdRetriever>();
248+
retriever->PutKey("kf", keys == Keys::WrongFooter ? SecureString(kNewFooterMasterKey)
249+
: kFooterEncryptionKey);
250+
if (keys != Keys::MissingColumn) {
251+
retriever->PutKey("kc1", kColumnEncryptionKey1);
252+
}
253+
retriever->PutKey("kc2", kColumnEncryptionKey2);
254+
255+
FileDecryptionProperties::Builder builder;
256+
builder.key_retriever(std::move(retriever));
257+
if (std::get<2>(GetParam()) != AadPrefixMode::None && prefix != AadPrefix::Missing) {
258+
builder.aad_prefix(prefix == AadPrefix::Wrong ? "incorrect-prefix" : kFileName);
259+
}
260+
return builder.build();
261+
}
262+
263+
std::unique_ptr<ParquetFileReader> OpenReader(
264+
std::shared_ptr<FileDecryptionProperties> decryption_properties) {
265+
auto reader_properties = default_reader_properties();
266+
reader_properties.file_decryption_properties(std::move(decryption_properties));
267+
return ParquetFileReader::OpenFile(file_path_, false, reader_properties);
268+
}
269+
270+
void CheckPlaintextFooterSignature() {
271+
PARQUET_ASSIGN_OR_THROW(auto file, ::arrow::io::ReadableFile::Open(file_path_));
272+
PARQUET_ASSIGN_OR_THROW(auto file_size, file->GetSize());
273+
ASSERT_GT(file_size, 8);
274+
PARQUET_ASSIGN_OR_THROW(auto trailer, file->ReadAt(file_size - 8, 8));
275+
ASSERT_EQ(trailer->size(), 8);
276+
const auto footer_size = ::arrow::bit_util::FromLittleEndian(
277+
::arrow::util::SafeLoadAs<uint32_t>(trailer->data()));
278+
constexpr auto signature_size = kNonceLength + kGcmTagLength;
279+
ASSERT_GT(footer_size, signature_size);
280+
ASSERT_LE(footer_size, file_size - 8);
281+
PARQUET_ASSIGN_OR_THROW(auto footer,
282+
file->ReadAt(file_size - 8 - footer_size, footer_size));
283+
ASSERT_EQ(footer->size(), footer_size);
284+
const auto footer_span = footer->span_as<uint8_t>();
285+
const auto metadata = footer_span.first(footer_size - signature_size);
286+
const auto nonce = footer_span.subspan(metadata.size(), kNonceLength);
287+
const auto tag = footer_span.last(kGcmTagLength);
288+
// Footer AAD is the file AAD followed by the footer module type byte.
289+
const auto aad = file_aad_ + static_cast<char>(kFooter);
290+
291+
// Verify the on-disk tag with GCM explicitly, independently of reader selection.
292+
auto encryptor = AesEncryptor::Make(ParquetCipher::AES_GCM_V1,
293+
static_cast<int32_t>(kFooterEncryptionKey.size()),
294+
/*metadata=*/true,
295+
/*write_length=*/false);
296+
std::vector<uint8_t> encrypted(encryptor->CiphertextLength(metadata.size()));
297+
encryptor->SignedFooterEncrypt(metadata, kFooterEncryptionKey.as_span(),
298+
str2span(aad), nonce, encrypted);
299+
EXPECT_EQ(std::vector<uint8_t>(tag.begin(), tag.end()),
300+
std::vector<uint8_t>(encrypted.end() - kGcmTagLength, encrypted.end()));
301+
}
302+
303+
void ReadFirstSecretValue(std::shared_ptr<FileDecryptionProperties> properties) {
304+
auto reader = OpenReader(std::move(properties));
305+
auto column = reader->RowGroup(0)->Column(5);
306+
auto* double_reader = static_cast<DoubleReader*>(column.get());
307+
double value;
308+
int64_t values_read;
309+
double_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
310+
}
311+
312+
std::unique_ptr<TemporaryDir> temp_dir_;
313+
std::string file_path_;
314+
std::string file_aad_;
315+
};
316+
317+
TEST_P(TestGeneratedEncryptionConfiguration, AlgorithmMetadataAndRoundTrip) {
318+
const auto cipher = std::get<0>(GetParam());
319+
const bool plaintext_footer = std::get<1>(GetParam());
320+
const auto prefix_mode = std::get<2>(GetParam());
321+
if (plaintext_footer) {
322+
CheckPlaintextFooterSignature();
323+
}
324+
auto reader = OpenReader(MakeDecryptionProperties());
325+
const auto metadata = reader->metadata();
326+
if (plaintext_footer) {
327+
ASSERT_TRUE(metadata->is_encryption_algorithm_set());
328+
const auto algorithm = metadata->encryption_algorithm();
329+
ASSERT_EQ(cipher, algorithm.algorithm);
330+
EXPECT_FALSE(algorithm.aad.aad_file_unique.empty());
331+
EXPECT_EQ(prefix_mode == AadPrefixMode::External, algorithm.aad.supply_aad_prefix);
332+
EXPECT_EQ(prefix_mode == AadPrefixMode::Stored ? kFileName : "",
333+
algorithm.aad.aad_prefix);
334+
} else {
335+
// An encrypted footer declares the algorithm in FileCryptoMetaData instead.
336+
EXPECT_FALSE(metadata->is_encryption_algorithm_set());
337+
}
338+
reader->Close();
339+
340+
FileDecryptor decryptor;
341+
// Verify every physical column and all row groups through sync and async readers.
342+
ASSERT_NO_THROW(decryptor.DecryptFile(file_path_, MakeDecryptionProperties()));
343+
if (prefix_mode == AadPrefixMode::Stored) {
344+
ASSERT_NO_THROW(decryptor.DecryptFile(
345+
file_path_, MakeDecryptionProperties(Keys::All, AadPrefix::Missing)));
346+
}
347+
if (plaintext_footer) {
348+
// This reads only public columns and does not authenticate the keyless footer.
349+
ASSERT_NO_THROW(decryptor.DecryptFile(file_path_, nullptr));
350+
}
351+
EXPECT_THROW(ReadFirstSecretValue(MakeDecryptionProperties(Keys::MissingColumn)),
352+
std::out_of_range);
353+
EXPECT_THROW(OpenReader(MakeDecryptionProperties(Keys::WrongFooter)), ParquetException);
354+
if (prefix_mode == AadPrefixMode::External) {
355+
EXPECT_THROW(OpenReader(MakeDecryptionProperties(Keys::All, AadPrefix::Missing)),
356+
ParquetException);
357+
EXPECT_THROW(OpenReader(MakeDecryptionProperties(Keys::All, AadPrefix::Wrong)),
358+
ParquetException);
359+
}
360+
}
361+
362+
INSTANTIATE_TEST_SUITE_P(
363+
GeneratedFiles, TestGeneratedEncryptionConfiguration,
364+
::testing::Combine(::testing::Values(ParquetCipher::AES_GCM_V1,
365+
ParquetCipher::AES_GCM_CTR_V1),
366+
::testing::Bool(),
367+
::testing::Values(AadPrefixMode::None, AadPrefixMode::Stored,
368+
AadPrefixMode::External)));
369+
198370
// Set temp_dir before running the write/read tests. The encrypted files will
199371
// be written/read from this directory.
200372
void TestEncryptionConfiguration::SetUpTestCase() {

‎cpp/src/parquet/metadata.cc‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -823,9 +823,11 @@ class FileMetaData::FileMetaDataImpl {
823823
const SecureString& key = file_decryptor_->GetFooterKey();
824824
const std::string& aad = encryption::CreateFooterAad(file_decryptor_->file_aad());
825825

826-
auto aes_encryptor = encryption::AesEncryptor::Make(file_decryptor_->algorithm(),
827-
static_cast<int>(key.size()),
828-
true, false /*write_length*/);
826+
// A plaintext footer is always signed with AES-GCM, regardless of the algorithm
827+
// configured for encryption.
828+
auto aes_encryptor = encryption::AesEncryptor::Make(
829+
ParquetCipher::AES_GCM_V1, static_cast<int>(key.size()), /*metadata=*/true,
830+
/*write_length=*/false);
829831

830832
std::shared_ptr<Buffer> encrypted_buffer = AllocateBuffer(
831833
file_decryptor_->pool(), aes_encryptor->CiphertextLength(
@@ -1202,8 +1204,10 @@ bool FileMetaData::VerifySignature(std::span<const uint8_t> serialized_metadata,
12021204
const SecureString& key = file_decryptor->GetFooterKey();
12031205
const std::string& aad = encryption::CreateFooterAad(file_decryptor->file_aad());
12041206

1207+
// A plaintext footer is always signed with AES-GCM, regardless of the algorithm
1208+
// configured for encryption.
12051209
auto aes_encryptor = encryption::AesEncryptor::Make(
1206-
file_decryptor->algorithm(), static_cast<int>(key.size()), /*metadata=*/true,
1210+
ParquetCipher::AES_GCM_V1, static_cast<int>(key.size()), /*metadata=*/true,
12071211
/*write_length=*/false);
12081212

12091213
std::shared_ptr<Buffer> encrypted_buffer =
@@ -2133,19 +2137,11 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl {
21332137
metadata_->column_orders.resize(schema_->num_columns(), column_order);
21342138
metadata_->__isset.column_orders = true;
21352139

2136-
// if plaintext footer, set footer signing algorithm
2140+
// Plaintext footers declare the file encryption algorithm and signing key metadata.
21372141
auto file_encryption_properties = properties_->file_encryption_properties();
21382142
if (file_encryption_properties && !file_encryption_properties->encrypted_footer()) {
2139-
EncryptionAlgorithm signing_algorithm;
2140-
EncryptionAlgorithm algo = file_encryption_properties->algorithm();
2141-
signing_algorithm.aad.aad_file_unique = algo.aad.aad_file_unique;
2142-
signing_algorithm.aad.supply_aad_prefix = algo.aad.supply_aad_prefix;
2143-
if (!algo.aad.supply_aad_prefix) {
2144-
signing_algorithm.aad.aad_prefix = algo.aad.aad_prefix;
2145-
}
2146-
signing_algorithm.algorithm = ParquetCipher::AES_GCM_V1;
2147-
2148-
metadata_->__set_encryption_algorithm(ToThrift(signing_algorithm));
2143+
metadata_->__set_encryption_algorithm(
2144+
ToThrift(file_encryption_properties->algorithm()));
21492145
const std::string& footer_signing_key_metadata =
21502146
file_encryption_properties->footer_key_metadata();
21512147
if (footer_signing_key_metadata.size() > 0) {

0 commit comments

Comments
 (0)