Skip to content

Commit 7f1f38d

Browse files
authored
GH-55406: [C++][Parquet] Test AES-CTR plaintext footer example file (#55407)
### Rationale for this change More test coverage. ### What changes are included in this PR? * Updates the `parquet-testing` submodule. * Adds a new test case for the test file added in apache/parquet-testing#130 * Allows regenerating the test files by setting a `PARQUET_ENCRYPTION_TEST_OUTPUT_DIR` environment variable when running the tests. This could be useful if we need to generate similar test files in the future. ### Are these changes tested? Yes ### Are there any user-facing changes? No ### Was AI used for this PR? In accordance to the [AI generation guidelines](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code), please disclose below whether and how AI was used in this PR. **PR code and description written by:** - [x] Human - [x] AI (code only, not description) **Reviewed before submission by:** - [x] Human - [ ] AI - [ ] Not reviewed * GitHub Issue: #55406 Authored-by: Adam Reeve <[email protected]> Signed-off-by: Adam Reeve <[email protected]>
1 parent 802102a commit 7f1f38d

5 files changed

Lines changed: 107 additions & 8 deletions

File tree

‎cpp/src/parquet/encryption/read_configurations_test.cc‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
* (instead of key retrieval callback).
5959
* - Decryption Configuration 4: PlainText Footer mode - test legacy reads,
6060
* read the footer + all non-encrypted columns.
61-
* (pairs with encryption configuration 3)
61+
* (pairs with encryption configurations 3 and 7)
6262
*
6363
* The encrypted parquet files that are read were encrypted using one of the
6464
* configurations below:
@@ -79,7 +79,9 @@
7979
* identity storage in file metadata.
8080
* - Encryption configuration 6: Encrypt two columns and the footer, with different
8181
* keys. Use the alternative (AES_GCM_CTR_V1) algorithm.
82-
82+
* - Encryption configuration 7: Encrypt two columns, with different keys.
83+
* Don't encrypt the footer.
84+
* Use the alternative (AES_GCM_CTR_V1) algorithm.
8385
*/
8486

8587
namespace parquet::encryption::test {
@@ -222,8 +224,10 @@ class TestDecryptionConfiguration
222224
}
223225
}
224226

225-
// decryption config 4 can only work when the encryption configuration is 3
226-
if (decryption_config_num == 4 && encryption_config_num != 3) {
227+
// decryption config 4 can only work with a plaintext footer (encryption
228+
// configurations 3 and 7)
229+
if (decryption_config_num == 4 && encryption_config_num != 3 &&
230+
encryption_config_num != 7) {
227231
return;
228232
}
229233
EXPECT_NO_THROW(DecryptFile(file_name, decryption_config_num - 1));
@@ -290,6 +294,7 @@ INSTANTIATE_TEST_SUITE_P(
290294
std::make_tuple(4, "encrypt_columns_and_footer_aad.parquet.encrypted"),
291295
std::make_tuple(
292296
5, "encrypt_columns_and_footer_disable_aad_storage.parquet.encrypted"),
293-
std::make_tuple(6, "encrypt_columns_and_footer_ctr.parquet.encrypted")));
297+
std::make_tuple(6, "encrypt_columns_and_footer_ctr.parquet.encrypted"),
298+
std::make_tuple(7, "encrypt_columns_plaintext_footer_ctr.parquet.encrypted")));
294299

295300
} // namespace parquet::encryption::test

‎cpp/src/parquet/encryption/test_encryption_util.cc‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,15 @@ void WriteBatch(int rows, const NextFunc get_next_column) {
175175

176176
FileEncryptor::FileEncryptor() { schema_ = SetupEncryptionSchema(); }
177177

178+
FileEncryptor FileEncryptor::ForParquetTestingFiles() {
179+
FileEncryptor encryptor;
180+
encryptor.num_rowgroups_ = 1;
181+
encryptor.compression_ = Compression::SNAPPY;
182+
encryptor.parquet_version_ = ParquetVersion::PARQUET_2_4;
183+
encryptor.rle_encode_boolean_ = true;
184+
return encryptor;
185+
}
186+
178187
std::shared_ptr<GroupNode> FileEncryptor::SetupEncryptionSchema() {
179188
parquet::schema::NodeVector fields;
180189

@@ -211,7 +220,11 @@ void FileEncryptor::EncryptFile(
211220
std::string file,
212221
std::shared_ptr<parquet::FileEncryptionProperties> encryption_configurations) {
213222
WriterProperties::Builder prop_builder;
214-
prop_builder.compression(parquet::Compression::UNCOMPRESSED);
223+
prop_builder.version(parquet_version_);
224+
prop_builder.compression(compression_);
225+
if (rle_encode_boolean_) {
226+
prop_builder.encoding(kBooleanFieldName, Encoding::RLE);
227+
}
215228
prop_builder.encryption(encryption_configurations);
216229
prop_builder.enable_write_page_index();
217230
std::shared_ptr<WriterProperties> writer_properties = prop_builder.build();

‎cpp/src/parquet/encryption/test_encryption_util.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "arrow/util/secure_string.h"
3535

3636
#include "parquet/encryption/encryption.h"
37+
#include "parquet/properties.h"
3738
#include "parquet/test_util.h"
3839

3940
namespace parquet {
@@ -102,6 +103,11 @@ class FileEncryptor {
102103
public:
103104
FileEncryptor();
104105

106+
// Create a FileEncryptor that writes files with the same layout as the encrypted
107+
// files in the parquet-testing repository (a single row group, Snappy compression
108+
// and RLE encoding for the boolean column).
109+
static FileEncryptor ForParquetTestingFiles();
110+
105111
void EncryptFile(
106112
std::string file,
107113
std::shared_ptr<parquet::FileEncryptionProperties> encryption_configurations);
@@ -111,6 +117,9 @@ class FileEncryptor {
111117

112118
int num_rowgroups_ = 5;
113119
int rows_per_rowgroup_ = 50;
120+
Compression::type compression_ = Compression::UNCOMPRESSED;
121+
ParquetVersion::type parquet_version_ = ParquetVersion::PARQUET_2_LATEST;
122+
bool rle_encode_boolean_ = false;
114123
std::shared_ptr<schema::GroupNode> schema_;
115124
};
116125

‎cpp/src/parquet/encryption/write_configurations_test.cc‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@
1919

2020
#include <stdio.h>
2121

22+
#include <fstream>
23+
#include <optional>
2224
#include <stdexcept>
2325
#include <tuple>
2426
#include <vector>
2527

2628
#include <arrow/io/file.h>
2729

30+
#include "arrow/util/config.h"
2831
#include "arrow/util/endian.h"
32+
#include "arrow/util/io_util.h"
2933
#include "arrow/util/ubsan.h"
3034

3135
#include "parquet/column_reader.h"
@@ -43,6 +47,11 @@
4347
* The files are saved in temporary folder and will be deleted after reading
4448
* them in read_configurations_test.cc test.
4549
*
50+
* To regenerate the corresponding files in the parquet-testing repository, set the
51+
* PARQUET_ENCRYPTION_TEST_OUTPUT_DIR environment variable to a directory. The files
52+
* are then written with the same layout as the parquet-testing files, and a copy of
53+
* each file (without the "tmp_" prefix) is written to that directory.
54+
*
4655
* A detailed description of the Parquet Modular Encryption specification can be found
4756
* here:
4857
* https://github.com/apache/parquet-format/blob/encryption/Encryption.md
@@ -66,6 +75,9 @@
6675
* identity storage in file metadata.
6776
* - Encryption configuration 6: Encrypt two columns and the footer, with different
6877
* keys. Use the alternative (AES_GCM_CTR_V1) algorithm.
78+
* - Encryption configuration 7: Encrypt two columns, with different keys.
79+
* Don't encrypt the footer.
80+
* Use the alternative (AES_GCM_CTR_V1) algorithm.
6981
*/
7082

7183
namespace parquet::encryption::test {
@@ -78,8 +90,19 @@ class TestEncryptionConfiguration : public ::testing::Test {
7890
public:
7991
static void SetUpTestCase();
8092

93+
void SetUp() override {
94+
#ifndef ARROW_WITH_SNAPPY
95+
if (output_dir_.has_value()) {
96+
FAIL() << "Generating parquet-testing files requires Snappy compression";
97+
}
98+
#endif
99+
}
100+
81101
protected:
82-
FileEncryptor encryptor_;
102+
// Set when generating files for the parquet-testing repository
103+
const std::optional<std::string> output_dir_ = GetOutputDir();
104+
FileEncryptor encryptor_ =
105+
output_dir_.has_value() ? FileEncryptor::ForParquetTestingFiles() : FileEncryptor();
83106

84107
std::string path_to_double_field_ = kDoubleFieldName;
85108
std::string path_to_float_field_ = kFloatFieldName;
@@ -93,6 +116,36 @@ class TestEncryptionConfiguration : public ::testing::Test {
93116
std::string file_name) {
94117
std::string file = temp_dir->path().ToString() + file_name;
95118
encryptor_.EncryptFile(file, encryption_configurations);
119+
CopyToOutputDir(file, file_name);
120+
}
121+
122+
static std::optional<std::string> GetOutputDir() {
123+
auto maybe_output_dir =
124+
::arrow::internal::GetEnvVar("PARQUET_ENCRYPTION_TEST_OUTPUT_DIR");
125+
if (!maybe_output_dir.ok()) {
126+
return std::nullopt;
127+
}
128+
return maybe_output_dir.MoveValueUnsafe();
129+
}
130+
131+
// Copy the file to the output directory if it is set, so the parquet-testing
132+
// files can be regenerated.
133+
void CopyToOutputDir(const std::string& file, const std::string& file_name) const {
134+
if (!output_dir_.has_value()) {
135+
return;
136+
}
137+
std::string output_name = file_name;
138+
const std::string tmp_prefix = "tmp_";
139+
if (output_name.rfind(tmp_prefix, 0) == 0) {
140+
output_name = output_name.substr(tmp_prefix.size());
141+
}
142+
std::ifstream src(file, std::ios::binary);
143+
std::ofstream dst(*output_dir_ + "/" + output_name, std::ios::binary);
144+
ASSERT_TRUE(src.good() && dst.good())
145+
<< "Failed to copy " << file << " to " << *output_dir_;
146+
dst << src.rdbuf();
147+
dst.close();
148+
ASSERT_TRUE(dst.good()) << "Failed to write " << output_name;
96149
}
97150
};
98151

@@ -203,6 +256,25 @@ TEST_F(TestEncryptionConfiguration, EncryptTwoColumnsAndFooterUseAES_GCM_CTR) {
203256
"tmp_encrypt_columns_and_footer_ctr.parquet.encrypted"));
204257
}
205258

259+
TEST_F(TestEncryptionConfiguration, EncryptTwoColumnsWithPlaintextFooterUseAES_GCM_CTR) {
260+
std::map<std::string, std::shared_ptr<ColumnEncryptionProperties>> encryption_cols7;
261+
encryption_cols7[path_to_double_field_] =
262+
ColumnEncryptionProperties::WithColumnKey(kColumnEncryptionKey1_, "kc1");
263+
encryption_cols7[path_to_float_field_] =
264+
ColumnEncryptionProperties::WithColumnKey(kColumnEncryptionKey2_, "kc2");
265+
encryption_cols7[kBooleanFieldName] = ColumnEncryptionProperties::Unencrypted();
266+
267+
FileEncryptionProperties::Builder file_encryption_builder_7(kFooterEncryptionKey_);
268+
269+
EXPECT_NO_THROW(
270+
this->EncryptFile(file_encryption_builder_7.footer_key_metadata("kf")
271+
->encrypted_columns(std::move(encryption_cols7))
272+
->set_plaintext_footer()
273+
->algorithm(parquet::ParquetCipher::AES_GCM_CTR_V1)
274+
->build(),
275+
"tmp_encrypt_columns_plaintext_footer_ctr.parquet.encrypted"));
276+
}
277+
206278
enum class AadPrefixMode { None, Stored, External };
207279

208280
class TestGeneratedEncryptionConfiguration

0 commit comments

Comments
 (0)