Skip to content

Commit 0793432

Browse files
tolleybotjorisvandenbosscheanjakefalakouwestonpace
authored
GH-29238 [C++][Dataset][Parquet] Support parquet modular encryption in the new Dataset API (#34616)
### Rationale for this change The purpose of this pull request is to support modular encryption in the new Dataset API. See [https://docs.google.com/document/d/13EysCNC6-Nu9wnJ8YpdzmD-aMLn4i2KXUJTNqIihy7A/edit#](url) for supporting document. ### What changes are included in this PR? I made improvements to the C++ and Python code to enable the Dataset API to have per-file settings for each file saved. Previously, the Dataset API applied the same encryption properties to all saved files, but now I've updated the code to allow for greater flexibility. In the Python code, I've added support for the changes by updating the ParquetFormat class to accept DatasetEncryptionConfiguration and DatasetDecryptionConfiguration structures. With these changes, you can pass the format object to the write_dataset function, giving you the ability to set unique encryption properties for each file in your Dataset. ### Are these changes tested? Yes, unit tests are included. I have also included a python sample project. ### Are there any user-facing changes? Yes, as stated above the ParquetFormat class has optional parameters for DatasetEncryptionConfiguration and DatasetDecryptionConfiguration through setters and getters. The Dataset now has the option using this to set different file encryption properties per file * Closes: #29238 Lead-authored-by: Don <[email protected]> Co-authored-by: Donald Tolley <[email protected]> Co-authored-by: Joris Van den Bossche <[email protected]> Co-authored-by: anjakefala <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Co-authored-by: Weston Pace <[email protected]> Co-authored-by: Gang Wu <[email protected]> Co-authored-by: scoder <[email protected]> Co-authored-by: Will Jones <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent b20e0ae commit 0793432

19 files changed

Lines changed: 1152 additions & 141 deletions

‎cpp/src/arrow/dataset/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,13 @@ endif()
175175

176176
if(ARROW_PARQUET)
177177
add_arrow_dataset_test(file_parquet_test)
178+
if(PARQUET_REQUIRE_ENCRYPTION AND ARROW_DATASET)
179+
add_arrow_dataset_test(file_parquet_encryption_test
180+
SOURCES
181+
file_parquet_encryption_test.cc
182+
${PROJECT_SOURCE_DIR}/src/parquet/encryption/test_in_memory_kms.cc
183+
)
184+
endif()
178185
endif()
179186

180187
if(ARROW_BUILD_BENCHMARKS)

‎cpp/src/arrow/dataset/file_parquet.cc‎

Lines changed: 67 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include "arrow/compute/exec.h"
2828
#include "arrow/dataset/dataset_internal.h"
29+
#include "arrow/dataset/parquet_encryption_config.h"
2930
#include "arrow/dataset/scanner.h"
3031
#include "arrow/filesystem/path_util.h"
3132
#include "arrow/table.h"
@@ -38,6 +39,9 @@
3839
#include "parquet/arrow/reader.h"
3940
#include "parquet/arrow/schema.h"
4041
#include "parquet/arrow/writer.h"
42+
#include "parquet/encryption/crypto_factory.h"
43+
#include "parquet/encryption/encryption.h"
44+
#include "parquet/encryption/kms_client.h"
4145
#include "parquet/file_reader.h"
4246
#include "parquet/properties.h"
4347
#include "parquet/statistics.h"
@@ -58,6 +62,7 @@ namespace {
5862

5963
parquet::ReaderProperties MakeReaderProperties(
6064
const ParquetFileFormat& format, ParquetFragmentScanOptions* parquet_scan_options,
65+
const std::string& path = "", std::shared_ptr<fs::FileSystem> filesystem = nullptr,
6166
MemoryPool* pool = default_memory_pool()) {
6267
// Can't mutate pool after construction
6368
parquet::ReaderProperties properties(pool);
@@ -67,8 +72,28 @@ parquet::ReaderProperties MakeReaderProperties(
6772
properties.disable_buffered_stream();
6873
}
6974
properties.set_buffer_size(parquet_scan_options->reader_properties->buffer_size());
75+
76+
#ifdef PARQUET_REQUIRE_ENCRYPTION
77+
auto parquet_decrypt_config = parquet_scan_options->parquet_decryption_config;
78+
79+
if (parquet_decrypt_config != nullptr) {
80+
auto file_decryption_prop =
81+
parquet_decrypt_config->crypto_factory->GetFileDecryptionProperties(
82+
*parquet_decrypt_config->kms_connection_config,
83+
*parquet_decrypt_config->decryption_config, path, filesystem);
84+
85+
parquet_scan_options->reader_properties->file_decryption_properties(
86+
std::move(file_decryption_prop));
87+
}
88+
#else
89+
if (parquet_scan_options->parquet_decryption_config != nullptr) {
90+
parquet::ParquetException::NYI("Encryption is not supported in this build.");
91+
}
92+
#endif
93+
7094
properties.file_decryption_properties(
7195
parquet_scan_options->reader_properties->file_decryption_properties());
96+
7297
properties.set_thrift_string_size_limit(
7398
parquet_scan_options->reader_properties->thrift_string_size_limit());
7499
properties.set_thrift_container_size_limit(
@@ -438,7 +463,7 @@ Result<std::shared_ptr<parquet::arrow::FileReader>> ParquetFileFormat::GetReader
438463
GetFragmentScanOptions<ParquetFragmentScanOptions>(kParquetTypeName, options.get(),
439464
default_fragment_scan_options));
440465
auto properties =
441-
MakeReaderProperties(*this, parquet_scan_options.get(), options->pool);
466+
MakeReaderProperties(*this, parquet_scan_options.get(), "", nullptr, options->pool);
442467
ARROW_ASSIGN_OR_RAISE(auto input, source.Open());
443468
// `parquet::ParquetFileReader::Open` will not wrap the exception as status,
444469
// so using `open_parquet_file` to wrap it.
@@ -477,9 +502,13 @@ Future<std::shared_ptr<parquet::arrow::FileReader>> ParquetFileFormat::GetReader
477502
auto parquet_scan_options,
478503
GetFragmentScanOptions<ParquetFragmentScanOptions>(kParquetTypeName, options.get(),
479504
default_fragment_scan_options));
480-
auto properties =
481-
MakeReaderProperties(*this, parquet_scan_options.get(), options->pool);
482-
505+
auto properties = MakeReaderProperties(*this, parquet_scan_options.get(), source.path(),
506+
source.filesystem(), options->pool);
507+
ARROW_ASSIGN_OR_RAISE(auto input, source.Open());
508+
// TODO(ARROW-12259): workaround since we have Future<(move-only type)>
509+
auto reader_fut = parquet::ParquetFileReader::OpenAsync(
510+
std::move(input), std::move(properties), metadata);
511+
auto path = source.path();
483512
auto self = checked_pointer_cast<const ParquetFileFormat>(shared_from_this());
484513

485514
return source.OpenAsync().Then(
@@ -666,10 +695,40 @@ Result<std::shared_ptr<FileWriter>> ParquetFileFormat::MakeWriter(
666695
auto parquet_options = checked_pointer_cast<ParquetFileWriteOptions>(options);
667696

668697
std::unique_ptr<parquet::arrow::FileWriter> parquet_writer;
669-
ARROW_ASSIGN_OR_RAISE(parquet_writer, parquet::arrow::FileWriter::Open(
670-
*schema, default_memory_pool(), destination,
671-
parquet_options->writer_properties,
672-
parquet_options->arrow_writer_properties));
698+
699+
#ifdef PARQUET_REQUIRE_ENCRYPTION
700+
auto parquet_encrypt_config = parquet_options->parquet_encryption_config;
701+
702+
if (parquet_encrypt_config != nullptr) {
703+
auto file_encryption_prop =
704+
parquet_encrypt_config->crypto_factory->GetFileEncryptionProperties(
705+
*parquet_encrypt_config->kms_connection_config,
706+
*parquet_encrypt_config->encryption_config, destination_locator.path,
707+
destination_locator.filesystem);
708+
709+
auto writer_properties =
710+
parquet::WriterProperties::Builder(*parquet_options->writer_properties)
711+
.encryption(std::move(file_encryption_prop))
712+
->build();
713+
714+
ARROW_ASSIGN_OR_RAISE(
715+
parquet_writer, parquet::arrow::FileWriter::Open(
716+
*schema, writer_properties->memory_pool(), destination,
717+
writer_properties, parquet_options->arrow_writer_properties));
718+
}
719+
#else
720+
if (parquet_options->parquet_encryption_config != nullptr) {
721+
return Status::NotImplemented("Encryption is not supported in this build.");
722+
}
723+
#endif
724+
725+
if (parquet_writer == nullptr) {
726+
ARROW_ASSIGN_OR_RAISE(parquet_writer,
727+
parquet::arrow::FileWriter::Open(
728+
*schema, parquet_options->writer_properties->memory_pool(),
729+
destination, parquet_options->writer_properties,
730+
parquet_options->arrow_writer_properties));
731+
}
673732

674733
return std::shared_ptr<FileWriter>(
675734
new ParquetFileWriter(std::move(destination), std::move(parquet_writer),

‎cpp/src/arrow/dataset/file_parquet.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ struct SchemaManifest;
5757
namespace arrow {
5858
namespace dataset {
5959

60+
struct ParquetDecryptionConfig;
61+
struct ParquetEncryptionConfig;
62+
6063
/// \addtogroup dataset-file-formats
6164
///
6265
/// @{
@@ -226,6 +229,8 @@ class ARROW_DS_EXPORT ParquetFragmentScanOptions : public FragmentScanOptions {
226229
/// ScanOptions. Additionally, dictionary columns come from
227230
/// ParquetFileFormat::ReaderOptions::dict_columns.
228231
std::shared_ptr<parquet::ArrowReaderProperties> arrow_reader_properties;
232+
/// A configuration structure that provides decryption properties for a dataset
233+
std::shared_ptr<ParquetDecryptionConfig> parquet_decryption_config = NULLPTR;
229234
};
230235

231236
class ARROW_DS_EXPORT ParquetFileWriteOptions : public FileWriteOptions {
@@ -236,6 +241,9 @@ class ARROW_DS_EXPORT ParquetFileWriteOptions : public FileWriteOptions {
236241
/// \brief Parquet Arrow writer properties.
237242
std::shared_ptr<parquet::ArrowWriterProperties> arrow_writer_properties;
238243

244+
// A configuration structure that provides encryption properties for a dataset
245+
std::shared_ptr<ParquetEncryptionConfig> parquet_encryption_config = NULLPTR;
246+
239247
protected:
240248
explicit ParquetFileWriteOptions(std::shared_ptr<FileFormat> format)
241249
: FileWriteOptions(std::move(format)) {}
Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <string_view>
19+
20+
#include "gtest/gtest.h"
21+
22+
#include <arrow/dataset/dataset.h>
23+
#include <arrow/dataset/file_base.h>
24+
#include <arrow/dataset/file_parquet.h>
25+
#include "arrow/array.h"
26+
#include "arrow/dataset/parquet_encryption_config.h"
27+
#include "arrow/dataset/partition.h"
28+
#include "arrow/filesystem/mockfs.h"
29+
#include "arrow/io/api.h"
30+
#include "arrow/status.h"
31+
#include "arrow/table.h"
32+
#include "arrow/testing/gtest_util.h"
33+
#include "arrow/type.h"
34+
#include "parquet/arrow/reader.h"
35+
#include "parquet/encryption/crypto_factory.h"
36+
#include "parquet/encryption/encryption.h"
37+
#include "parquet/encryption/kms_client.h"
38+
#include "parquet/encryption/test_in_memory_kms.h"
39+
40+
constexpr std::string_view kFooterKeyMasterKey = "0123456789012345";
41+
constexpr std::string_view kFooterKeyMasterKeyId = "footer_key";
42+
constexpr std::string_view kFooterKeyName = "footer_key";
43+
constexpr std::string_view kColumnMasterKey = "1234567890123450";
44+
constexpr std::string_view kColumnMasterKeyId = "col_key";
45+
constexpr std::string_view kColumnKeyMapping = "col_key: a";
46+
constexpr std::string_view kBaseDir = "";
47+
48+
using arrow::internal::checked_pointer_cast;
49+
50+
namespace arrow {
51+
namespace dataset {
52+
53+
class DatasetEncryptionTest : public ::testing::Test {
54+
public:
55+
// This function creates a mock file system using the current time point, creates a
56+
// directory with the given base directory path, and writes a dataset to it using
57+
// provided Parquet file write options. The dataset is partitioned using a Hive
58+
// partitioning scheme. The function also checks if the written files exist in the file
59+
// system.
60+
static void SetUpTestSuite() {
61+
// Creates a mock file system using the current time point.
62+
EXPECT_OK_AND_ASSIGN(file_system_, fs::internal::MockFileSystem::Make(
63+
std::chrono::system_clock::now(), {}));
64+
ASSERT_OK(file_system_->CreateDir(std::string(kBaseDir)));
65+
66+
// Prepare table data.
67+
auto table_schema = schema({field("a", int64()), field("c", int64()),
68+
field("e", int64()), field("part", utf8())});
69+
table_ = TableFromJSON(table_schema, {R"([
70+
[ 0, 9, 1, "a" ],
71+
[ 1, 8, 2, "a" ],
72+
[ 2, 7, 1, "c" ],
73+
[ 3, 6, 2, "c" ],
74+
[ 4, 5, 1, "e" ],
75+
[ 5, 4, 2, "e" ],
76+
[ 6, 3, 1, "g" ],
77+
[ 7, 2, 2, "g" ],
78+
[ 8, 1, 1, "i" ],
79+
[ 9, 0, 2, "i" ]
80+
])"});
81+
82+
// Use a Hive-style partitioning scheme.
83+
partitioning_ = std::make_shared<HivePartitioning>(schema({field("part", utf8())}));
84+
85+
// Prepare encryption properties.
86+
std::unordered_map<std::string, std::string> key_map;
87+
key_map.emplace(kColumnMasterKeyId, kColumnMasterKey);
88+
key_map.emplace(kFooterKeyMasterKeyId, kFooterKeyMasterKey);
89+
90+
crypto_factory_ = std::make_shared<parquet::encryption::CryptoFactory>();
91+
auto kms_client_factory =
92+
std::make_shared<parquet::encryption::TestOnlyInMemoryKmsClientFactory>(
93+
/*wrap_locally=*/true, key_map);
94+
crypto_factory_->RegisterKmsClientFactory(std::move(kms_client_factory));
95+
kms_connection_config_ = std::make_shared<parquet::encryption::KmsConnectionConfig>();
96+
97+
// Set write options with encryption configuration.
98+
auto encryption_config =
99+
std::make_shared<parquet::encryption::EncryptionConfiguration>(
100+
std::string(kFooterKeyName));
101+
encryption_config->column_keys = kColumnKeyMapping;
102+
auto parquet_encryption_config = std::make_shared<ParquetEncryptionConfig>();
103+
// Directly assign shared_ptr objects to ParquetEncryptionConfig members
104+
parquet_encryption_config->crypto_factory = crypto_factory_;
105+
parquet_encryption_config->kms_connection_config = kms_connection_config_;
106+
parquet_encryption_config->encryption_config = std::move(encryption_config);
107+
108+
auto file_format = std::make_shared<ParquetFileFormat>();
109+
auto parquet_file_write_options =
110+
checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions());
111+
parquet_file_write_options->parquet_encryption_config =
112+
std::move(parquet_encryption_config);
113+
114+
// Write dataset.
115+
auto dataset = std::make_shared<InMemoryDataset>(table_);
116+
EXPECT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
117+
EXPECT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
118+
119+
FileSystemDatasetWriteOptions write_options;
120+
write_options.file_write_options = parquet_file_write_options;
121+
write_options.filesystem = file_system_;
122+
write_options.base_dir = kBaseDir;
123+
write_options.partitioning = partitioning_;
124+
write_options.basename_template = "part{i}.parquet";
125+
ASSERT_OK(FileSystemDataset::Write(write_options, std::move(scanner)));
126+
}
127+
128+
protected:
129+
inline static std::shared_ptr<fs::FileSystem> file_system_;
130+
inline static std::shared_ptr<Table> table_;
131+
inline static std::shared_ptr<HivePartitioning> partitioning_;
132+
inline static std::shared_ptr<parquet::encryption::CryptoFactory> crypto_factory_;
133+
inline static std::shared_ptr<parquet::encryption::KmsConnectionConfig>
134+
kms_connection_config_;
135+
};
136+
137+
// This test demonstrates the process of writing a partitioned Parquet file with the same
138+
// encryption properties applied to each file within the dataset. The encryption
139+
// properties are determined based on the selected columns. After writing the dataset, the
140+
// test reads the data back and verifies that it can be successfully decrypted and
141+
// scanned.
142+
TEST_F(DatasetEncryptionTest, WriteReadDatasetWithEncryption) {
143+
// Create decryption properties.
144+
auto decryption_config =
145+
std::make_shared<parquet::encryption::DecryptionConfiguration>();
146+
auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
147+
parquet_decryption_config->crypto_factory = crypto_factory_;
148+
parquet_decryption_config->kms_connection_config = kms_connection_config_;
149+
parquet_decryption_config->decryption_config = std::move(decryption_config);
150+
151+
// Set scan options.
152+
auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
153+
parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config);
154+
155+
auto file_format = std::make_shared<ParquetFileFormat>();
156+
file_format->default_fragment_scan_options = std::move(parquet_scan_options);
157+
158+
// Get FileInfo objects for all files under the base directory
159+
fs::FileSelector selector;
160+
selector.base_dir = kBaseDir;
161+
selector.recursive = true;
162+
163+
FileSystemFactoryOptions factory_options;
164+
factory_options.partitioning = partitioning_;
165+
factory_options.partition_base_dir = kBaseDir;
166+
ASSERT_OK_AND_ASSIGN(auto dataset_factory,
167+
FileSystemDatasetFactory::Make(file_system_, selector, file_format,
168+
factory_options));
169+
170+
// Read dataset into table
171+
ASSERT_OK_AND_ASSIGN(auto dataset, dataset_factory->Finish());
172+
ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
173+
ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
174+
ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable());
175+
176+
// Verify the data was read correctly
177+
ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks());
178+
// Validate the table
179+
ASSERT_OK(combined_table->ValidateFull());
180+
AssertTablesEqual(*combined_table, *table_);
181+
}
182+
183+
// Read a single parquet file with and without decryption properties.
184+
TEST_F(DatasetEncryptionTest, ReadSingleFile) {
185+
// Open the Parquet file.
186+
ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile("part=a/part0.parquet"));
187+
188+
// Try to read metadata without providing decryption properties
189+
// when the footer is encrypted.
190+
ASSERT_THROW(parquet::ReadMetaData(input), parquet::ParquetException);
191+
192+
// Create the ReaderProperties object using the FileDecryptionProperties object
193+
auto decryption_config =
194+
std::make_shared<parquet::encryption::DecryptionConfiguration>();
195+
auto file_decryption_properties = crypto_factory_->GetFileDecryptionProperties(
196+
*kms_connection_config_, *decryption_config);
197+
auto reader_properties = parquet::default_reader_properties();
198+
reader_properties.file_decryption_properties(file_decryption_properties);
199+
200+
// Read entire file as a single Arrow table
201+
parquet::arrow::FileReaderBuilder reader_builder;
202+
ASSERT_OK(reader_builder.Open(input, reader_properties));
203+
ASSERT_OK_AND_ASSIGN(auto arrow_reader, reader_builder.Build());
204+
std::shared_ptr<Table> table;
205+
ASSERT_OK(arrow_reader->ReadTable(&table));
206+
207+
// Check the contents of the table
208+
ASSERT_EQ(table->num_rows(), 2);
209+
ASSERT_EQ(table->num_columns(), 3);
210+
ASSERT_EQ(checked_pointer_cast<Int64Array>(table->column(0)->chunk(0))->GetView(0), 0);
211+
ASSERT_EQ(checked_pointer_cast<Int64Array>(table->column(1)->chunk(0))->GetView(0), 9);
212+
ASSERT_EQ(checked_pointer_cast<Int64Array>(table->column(2)->chunk(0))->GetView(0), 1);
213+
}
214+
215+
} // namespace dataset
216+
} // namespace arrow

0 commit comments

Comments
 (0)