Skip to content

Commit d1dce12

Browse files
committed
Initial commit
diff --git c/cpp/src/arrow/dataset/file_parquet.cc i/cpp/src/arrow/dataset/file_parquet.cc index fd601b6..7ebf1ee 100644 --- c/cpp/src/arrow/dataset/file_parquet.cc +++ i/cpp/src/arrow/dataset/file_parquet.cc @@ -35,6 +35,8 @@ #include "arrow/util/future.h" #include "arrow/util/iterator.h" #include "arrow/util/logging_internal.h" +#include "arrow/util/key_value_metadata.h" +#include "arrow/util/logging.h" #include "arrow/util/range.h" #include "arrow/util/tracing_internal.h" #include "parquet/arrow/reader.h" @@ -42,6 +44,7 @@ #include "parquet/arrow/writer.h" #include "parquet/encryption/crypto_factory.h" #include "parquet/encryption/encryption.h" +#include "parquet/encryption/internal_file_decryptor.h" #include "parquet/encryption/kms_client.h" #include "parquet/file_reader.h" #include "parquet/properties.h" @@ -1091,7 +1094,17 @@ Result<std::shared_ptr<DatasetFactory>> ParquetDatasetFactory::Make( std::vector<std::pair<std::string, std::vector<int>>> paths_with_row_group_ids; std::unordered_map<std::string, int> paths_to_index; + // Check if file is a metadata only file and if so extract the AADs for each RowGroup + auto kvmd = metadata->key_value_metadata(); + auto encrypted_metadata_only_file = + metadata_source.path() == "_metadata" && kvmd && kvmd->Contains("row_group_aad_0"); + for (int i = 0; i < metadata->num_row_groups(); i++) { + if (encrypted_metadata_only_file) { + PARQUET_ASSIGN_OR_THROW(auto aad, metadata->key_value_metadata()->Get( + "row_group_aad_" + std::to_string(i))); + metadata->set_file_decryptor_aad(aad); + } auto row_group = metadata->RowGroup(i); ARROW_ASSIGN_OR_RAISE(auto path, FileFromRowGroup(filesystem.get(), base_path, *row_group, diff --git c/cpp/src/arrow/dataset/file_parquet_encryption_test.cc i/cpp/src/arrow/dataset/file_parquet_encryption_test.cc index d2e1763..0181570 100644 --- c/cpp/src/arrow/dataset/file_parquet_encryption_test.cc +++ i/cpp/src/arrow/dataset/file_parquet_encryption_test.cc @@ -35,11 +35,14 @@ #include "arrow/type.h" #include "arrow/util/future.h" #include "arrow/util/thread_pool.h" +#include "arrow/util/key_value_metadata.h" #include "parquet/arrow/reader.h" +#include "parquet/arrow/schema.h" #include "parquet/encryption/crypto_factory.h" #include "parquet/encryption/encryption_internal.h" #include "parquet/encryption/kms_client.h" #include "parquet/encryption/test_in_memory_kms.h" +#include "parquet/file_writer.h" constexpr std::string_view kFooterKeyMasterKey = "0123456789012345"; constexpr std::string_view kFooterKeyMasterKeyId = "footer_key"; @@ -54,6 +57,62 @@ using arrow::internal::checked_pointer_cast; namespace arrow { namespace dataset { +class DatasetTestBase : public ::testing::Test { + public: + void SetUp() override { + // Creates a mock file system using the current time point. + EXPECT_OK_AND_ASSIGN(file_system_, fs::internal::MockFileSystem::Make( + std::chrono::system_clock::now(), {})); + ASSERT_OK(file_system_->CreateDir(std::string(kBaseDir))); + + // Init dataset and partitioning. + ASSERT_NO_FATAL_FAILURE(PrepareTableAndPartitioning()); + + auto file_format = std::make_shared<ParquetFileFormat>(); + auto parquet_file_write_options = + checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions()); + + // Write dataset. + auto dataset = std::make_shared<InMemoryDataset>(table_); + EXPECT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan()); + EXPECT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish()); + + FileSystemDatasetWriteOptions write_options; + write_options.file_write_options = parquet_file_write_options; + write_options.filesystem = file_system_; + write_options.base_dir = kBaseDir; + write_options.partitioning = partitioning_; + write_options.basename_template = "part{i}.parquet"; + ASSERT_OK(FileSystemDataset::Write(write_options, std::move(scanner))); + } + + void PrepareTableAndPartitioning() { + // Prepare table data. + auto table_schema = schema({field("a", int64()), field("c", int64()), + field("e", int64()), field("part", utf8())}); + table_ = TableFromJSON(table_schema, {R"([ + [ 0, 9, 1, "a" ], + [ 1, 8, 2, "a" ], + [ 2, 7, 1, "a" ], + [ 3, 6, 2, "a" ], + [ 4, 5, 1, "a" ], + [ 5, 4, 2, "a" ], + [ 6, 3, 1, "b" ], + [ 7, 2, 2, "b" ], + [ 8, 1, 1, "b" ], + [ 9, 0, 2, "b" ] + ])"}); + + // Use a Hive-style partitioning scheme. + partitioning_ = std::make_shared<HivePartitioning>(schema({field("part", utf8())})); + } + + protected: + std::shared_ptr<fs::FileSystem> file_system_; + std::shared_ptr<Table> table_; + std::shared_ptr<Partitioning> partitioning_; +}; + struct EncryptionTestParam { bool uniform_encryption; // false is using per-column keys bool concurrently; @@ -393,6 +452,180 @@ TEST_P(DatasetEncryptionTest, ReadSingleFile) { INSTANTIATE_TEST_SUITE_P(DatasetEncryptionTest, DatasetEncryptionTest, kAllParamValues); +Result<std::vector<std::shared_ptr<parquet::FileMetaData>>> ReadMetadata( + const std::vector<std::string>& paths, std::shared_ptr<fs::FileSystem>& file_system, + const parquet::ReaderProperties& reader_properties) { + if (paths.empty()) { + return Status::Invalid("No files to read metadata from"); + } + + std::vector<std::shared_ptr<parquet::FileMetaData>> metadata; + + for (const auto& path : paths) { + PARQUET_ASSIGN_OR_THROW(auto input, file_system->OpenInputFile(path)); + auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties); + auto file_metadata = parquet_reader->metadata(); + file_metadata->set_file_path(path); + metadata.push_back(file_metadata); + } + return metadata; +} + +// Write encrypted _metadata file and read the dataset using the metadata. +TEST_F(DatasetEncryptionTest, ReadDatasetFromEncryptedMetadata) { + // Create decryption properties. + auto decryption_config = + std::make_shared<parquet::encryption::DecryptionConfiguration>(); + auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>(); + parquet_decryption_config->crypto_factory = crypto_factory_; + parquet_decryption_config->kms_connection_config = kms_connection_config_; + parquet_decryption_config->decryption_config = std::move(decryption_config); + + // Set scan options. + auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>(); + parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config); + + auto file_format = std::make_shared<ParquetFileFormat>(); + file_format->default_fragment_scan_options = std::move(parquet_scan_options); + + auto reader_properties = parquet::default_reader_properties(); + decryption_config = std::make_shared<parquet::encryption::DecryptionConfiguration>(); + reader_properties.file_decryption_properties( + crypto_factory_->GetFileDecryptionProperties(*kms_connection_config_, + *decryption_config)); + auto encryption_config = std::make_shared<parquet::encryption::EncryptionConfiguration>( + std::string(kFooterKeyName)); + encryption_config->column_keys = kColumnKeyMapping; + + auto parquet_encryption_config = std::make_shared<ParquetEncryptionConfig>(); + // Directly assign shared_ptr objects to ParquetEncryptionConfig members + parquet_encryption_config->crypto_factory = crypto_factory_; + parquet_encryption_config->kms_connection_config = kms_connection_config_; + parquet_encryption_config->encryption_config = encryption_config; + + auto parquet_file_write_options = + checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions()); + parquet_file_write_options->parquet_encryption_config = + std::move(parquet_encryption_config); + + std::vector<std::string> paths = {"part=a/part0.parquet", "part=c/part0.parquet", + "part=e/part0.parquet", "part=g/part0.parquet", + "part=i/part0.parquet"}; + + auto file_encryption_properties = crypto_factory_->GetFileEncryptionProperties( + *kms_connection_config_, *encryption_config); + + auto table_schema = + schema({field("a", int64()), field("c", int64()), field("e", int64())}); + std::shared_ptr<parquet::SchemaDescriptor> schema; + auto writer_props = parquet::WriterProperties::Builder().build(); + + PARQUET_ASSIGN_OR_THROW(auto metadata_list, + ReadMetadata(paths, file_system_, reader_properties)); + PARQUET_ASSIGN_OR_THROW(auto metadata, parquet::FileMetaData::CoalesceMetadata( + metadata_list, writer_props)); + + // TODO: Make sure plaintext footer mode works + // encryption_config->plaintext_footer = true; + file_encryption_properties = crypto_factory_->GetFileEncryptionProperties( + *kms_connection_config_, *encryption_config); + + // Write metadata to _metadata file. + std::string metadata_path = "_metadata"; + ASSERT_OK_AND_ASSIGN(auto stream, file_system_->OpenOutputStream(metadata_path)); + WriteEncryptedMetadataFile(*metadata, stream, file_encryption_properties); + ARROW_EXPECT_OK(stream->Close()); + + // Set scan options. + decryption_config = std::make_shared<parquet::encryption::DecryptionConfiguration>(); + parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>(); + parquet_decryption_config->crypto_factory = crypto_factory_; + parquet_decryption_config->kms_connection_config = kms_connection_config_; + parquet_decryption_config->decryption_config = std::move(decryption_config); + + parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>(); + parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config); + + file_format = std::make_shared<ParquetFileFormat>(); + file_format->default_fragment_scan_options = std::move(parquet_scan_options); + + ParquetFactoryOptions parquet_factory_options; + parquet_factory_options.partitioning = partitioning_; + parquet_factory_options.partition_base_dir = kBaseDir; + + ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile(metadata_path)); + auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties); + auto file_metadata = parquet_reader->metadata(); + + ASSERT_TRUE(file_metadata->Equals(*metadata)); + + // Create parquet dataset factory + ASSERT_OK_AND_ASSIGN(auto parquet_dataset_factory, + ParquetDatasetFactory::Make(metadata_path, file_system_, + file_format, parquet_factory_options)); + + // Create the dataset + ASSERT_OK_AND_ASSIGN(auto dataset, parquet_dataset_factory->Finish()); + + // Read dataset into table + ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan()); + ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish()); + ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable()); + + // Verify the data was read correctly + ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks()); + + // Validate the table + ASSERT_OK(combined_table->ValidateFull()); + AssertTablesEqual(*table_, *combined_table); +} + +// Write _metadata file and read the dataset using the metadata. +TEST_F(DatasetTestBase, ReadDatasetFromMetadata) { + auto reader_properties = parquet::default_reader_properties(); + + std::vector<std::string> paths = {"part=a/part0.parquet", "part=b/part0.parquet"}; + std::vector<std::shared_ptr<parquet::FileMetaData>> metadata; + + for (const auto& path : paths) { + ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile(path)); + auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties); + auto file_metadata = parquet_reader->metadata(); + // Make sure file_paths are stored in metadata + file_metadata->set_file_path(path); + metadata.push_back(file_metadata); + } + metadata[0]->AppendRowGroups(*metadata[1]); + + std::string metadata_path = "_metadata"; + ASSERT_OK_AND_ASSIGN(auto stream, file_system_->OpenOutputStream(metadata_path)); + WriteMetaDataFile(*metadata[0], stream.get()); + ARROW_EXPECT_OK(stream->Close()); + + auto file_format = std::make_shared<ParquetFileFormat>(); + ParquetFactoryOptions factory_options; + factory_options.partitioning = partitioning_; + factory_options.partition_base_dir = kBaseDir; + ASSERT_OK_AND_ASSIGN(auto dataset_factory, + ParquetDatasetFactory::Make(metadata_path, file_system_, + file_format, factory_options)); + + // Create the dataset + ASSERT_OK_AND_ASSIGN(auto dataset, dataset_factory->Finish()); + + // Read dataset into table + ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan()); + ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish()); + ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable()); + + // Verify the data was read correctly + ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks()); + + // Validate the table + ASSERT_OK(combined_table->ValidateFull()); + AssertTablesEqual(*combined_table, *table_); +} + // apacheGH-39444: This test covers the case where parquet dataset scanner crashes when // processing encrypted datasets over 2^15 rows in multi-threaded mode. class LargeRowCountEncryptionTest : public DatasetEncryptionTestBase { diff --git c/cpp/src/parquet/arrow/writer.cc i/cpp/src/parquet/arrow/writer.cc index 5307465..43ac9a6 100644 --- c/cpp/src/parquet/arrow/writer.cc +++ i/cpp/src/parquet/arrow/writer.cc @@ -589,6 +589,14 @@ Status WriteMetaDataFile(const FileMetaData& file_metadata, return Status::OK(); } +Status WriteEncryptedMetadataFile( + const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink, + std::shared_ptr<FileEncryptionProperties> file_encryption_properties) { + PARQUET_CATCH_NOT_OK(::parquet::WriteEncryptedMetadataFile(file_metadata, sink, + file_encryption_properties)); + return Status::OK(); +} + Status WriteTable(const ::arrow::Table& table, ::arrow::MemoryPool* pool, std::shared_ptr<::arrow::io::OutputStream> sink, int64_t chunk_size, std::shared_ptr<WriterProperties> properties, diff --git c/cpp/src/parquet/arrow/writer.h i/cpp/src/parquet/arrow/writer.h index 8ec8796..194d91f 100644 --- c/cpp/src/parquet/arrow/writer.h +++ i/cpp/src/parquet/arrow/writer.h @@ -153,6 +153,12 @@ PARQUET_EXPORT ::arrow::Status WriteMetaDataFile(const FileMetaData& file_metadata, ::arrow::io::OutputStream* sink); +/// \brief Write encrypted metadata-only Parquet file to indicated Arrow OutputStream +PARQUET_EXPORT +::arrow::Status WriteEncryptedMetadataFile( + const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink, + std::shared_ptr<FileEncryptionProperties> file_encryption_properties); + /// \brief Write a Table to Parquet. /// /// This writes one table in a single shot. To write a Parquet file with diff --git c/cpp/src/parquet/file_writer.cc i/cpp/src/parquet/file_writer.cc index 498fda4..3a35fbd 100644 --- c/cpp/src/parquet/file_writer.cc +++ i/cpp/src/parquet/file_writer.cc @@ -583,6 +583,46 @@ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata, } } +void WriteEncryptedMetadataFile( + const FileMetaData& metadata, std::shared_ptr<::arrow::io::OutputStream> sink, + std::shared_ptr<FileEncryptionProperties> file_encryption_properties) { + auto file_encryptor = std::make_unique<InternalFileEncryptor>( + file_encryption_properties.get(), ::arrow::default_memory_pool()); + + if (file_encryption_properties->encrypted_footer()) { + PARQUET_THROW_NOT_OK(sink->Write(kParquetEMagic, 4)); + + PARQUET_ASSIGN_OR_THROW(int64_t position, sink->Tell()); + auto metadata_start = static_cast<uint64_t>(position); + + auto writer_props = parquet::WriterProperties::Builder() + .encryption(file_encryption_properties) + ->build(); + auto builder = FileMetaDataBuilder::Make(metadata.schema(), writer_props); + + auto footer_metadata = builder->Finish(metadata.key_value_metadata()); + auto crypto_metadata = builder->GetCryptoMetaData(); + + WriteFileCryptoMetaData(*crypto_metadata, sink.get()); + + auto footer_encryptor = file_encryptor->GetFooterEncryptor(); + WriteEncryptedFileMetadata(metadata, sink.get(), footer_encryptor, true); + + PARQUET_ASSIGN_OR_THROW(position, sink->Tell()); + auto footer_and_crypto_len = static_cast<uint32_t>(position - metadata_start); + PARQUET_THROW_NOT_OK( + sink->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4)); + PARQUET_THROW_NOT_OK(sink->Write(kParquetEMagic, 4)); + } else { + // Encrypted file with plaintext footer mode. + PARQUET_THROW_NOT_OK(sink->Write(kParquetMagic, 4)); + auto footer_signing_encryptor = file_encryptor->GetFooterSigningEncryptor(); + WriteEncryptedFileMetadata(metadata, sink.get(), footer_signing_encryptor, false); + } + + file_encryptor->WipeOutEncryptionKeys(); +} + void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata, ArrowOutputStream* sink) { crypto_metadata.WriteTo(sink); diff --git c/cpp/src/parquet/file_writer.h i/cpp/src/parquet/file_writer.h index d5ea1d7..60b8474 100644 --- c/cpp/src/parquet/file_writer.h +++ i/cpp/src/parquet/file_writer.h @@ -117,6 +117,11 @@ PARQUET_EXPORT void WriteMetaDataFile(const FileMetaData& file_metadata, ::arrow::io::OutputStream* sink); +PARQUET_EXPORT +void WriteEncryptedMetadataFile( + const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink, + std::shared_ptr<FileEncryptionProperties> file_encryption_properties); + PARQUET_EXPORT void WriteEncryptedFileMetadata(const FileMetaData& file_metadata, ArrowOutputStream* sink, @@ -125,9 +130,10 @@ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata, PARQUET_EXPORT void WriteEncryptedFileMetadata(const FileMetaData& file_metadata, - ::arrow::io::OutputStream* sink, + std::shared_ptr<::arrow::io::OutputStream> sink, const std::shared_ptr<Encryptor>& encryptor = NULLPTR, bool encrypt_footer = false); + PARQUET_EXPORT void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata, ::arrow::io::OutputStream* sink); diff --git c/cpp/src/parquet/metadata.cc i/cpp/src/parquet/metadata.cc index 9fbf8f1..ea8e4f1 100644 --- c/cpp/src/parquet/metadata.cc +++ i/cpp/src/parquet/metadata.cc @@ -905,6 +905,13 @@ class FileMetaData::FileMetaDataImpl { out->impl_->key_value_metadata_ = key_value_metadata_; out->impl_->file_decryptor_ = file_decryptor_; + if (key_value_metadata_ && key_value_metadata_->Contains("row_group_aad_0")) { + PARQUET_ASSIGN_OR_THROW( + auto aad, + key_value_metadata_->Get("row_group_aad_" + std::to_string(row_groups[0]))); + out->set_file_decryptor_aad(aad); + } + return out; } @@ -1029,6 +1036,10 @@ EncryptionAlgorithm FileMetaData::encryption_algorithm() const { return impl_->encryption_algorithm(); } +const std::string& FileMetaData::file_aad() const { + return impl_->file_decryptor()->file_aad(); +} + const std::string& FileMetaData::footer_signing_key_metadata() const { return impl_->footer_signing_key_metadata(); } @@ -1038,6 +1049,13 @@ void FileMetaData::set_file_decryptor( impl_->set_file_decryptor(std::move(file_decryptor)); } +void FileMetaData::set_file_decryptor_aad(const std::string& aad) { + auto fd = impl_->file_decryptor(); + auto file_decryptor = std::make_shared<parquet::InternalFileDecryptor>( + fd->properties(), aad, fd->algorithm(), fd->footer_key_metadata(), fd->pool()); + set_file_decryptor(file_decryptor); +} + const std::shared_ptr<InternalFileDecryptor>& FileMetaData::file_decryptor() const { return impl_->file_decryptor(); } @@ -1089,6 +1107,36 @@ void FileMetaData::WriteTo(::arrow::io::OutputStream* dst, return impl_->WriteTo(dst, encryptor); } +::arrow::Result<std::shared_ptr<parquet::FileMetaData>> FileMetaData::CoalesceMetadata( + std::vector<std::shared_ptr<parquet::FileMetaData>>& metadata_list, + std::shared_ptr<parquet::WriterProperties>& writer_props) { + if (metadata_list.empty()) { + return ::arrow::Status::Invalid("No metadata to coalesce"); + } + + std::vector<std::string> values, keys; + + // Read metadata from all dataset files and store AADs. + for (size_t i = 0; i < metadata_list.size(); i++) { + const auto& file_metadata = metadata_list[i]; + keys.push_back("row_group_aad_" + std::to_string(i)); + values.push_back(file_metadata->file_aad()); + if (i > 0) { + metadata_list[0]->AppendRowGroups(*file_metadata); + } + } + + // Create a new FileMetadata object with the created AADs as key_value_metadata. + auto fmd_builder = + parquet::FileMetaDataBuilder::Make(metadata_list[0]->schema(), writer_props); + const std::shared_ptr<const KeyValueMetadata> file_aad_metadata = + ::arrow::key_value_metadata(keys, values); + auto metadata = fmd_builder->Finish(file_aad_metadata); + metadata->AppendRowGroups(*metadata_list[0]); + + return metadata; +} + class FileCryptoMetaData::FileCryptoMetaDataImpl { public: FileCryptoMetaDataImpl() = default; diff --git c/cpp/src/parquet/metadata.h i/cpp/src/parquet/metadata.h index 9a3964f..378c1f3 100644 --- c/cpp/src/parquet/metadata.h +++ i/cpp/src/parquet/metadata.h @@ -328,6 +328,8 @@ class PARQUET_EXPORT FileMetaData { EncryptionAlgorithm encryption_algorithm() const; const std::string& footer_signing_key_metadata() const; + const std::string& file_aad() const; + /// \brief Verify signature of FileMetaData when file is encrypted but footer /// is not encrypted (plaintext footer). bool VerifySignature(const void* signature); @@ -377,6 +379,23 @@ class PARQUET_EXPORT FileMetaData { /// debug text (if true). std::string SerializeUnencrypted(bool scrub, bool debug) const; + /// \brief Coalesce metadata from multiple files into a single metadata file by + /// appending row groups. + /// + /// \param[in] metadata_list list of FileMetaData objects to coalesce. + /// \param[in] writer_props WriterProperties to use to create coalesced FileMetaData + /// object. + /// \return a new FileMetaData object containing all row groups from the input + /// FileMetaData objects. + static ::arrow::Result<std::shared_ptr<FileMetaData>> CoalesceMetadata( + std::vector<std::shared_ptr<FileMetaData>>& metadata_list, + std::shared_ptr<WriterProperties>& writer_props); + + /// \brief Set the AAD of decryptor of the file. + /// + /// \param[in] aad AAD to set on the decryptor. + void set_file_decryptor_aad(const std::string& aad); + private: friend FileMetaDataBuilder; friend class SerializedFile;
1 parent 586ed92 commit d1dce12

8 files changed

Lines changed: 372 additions & 1 deletion

File tree

‎cpp/src/arrow/dataset/file_parquet.cc‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,16 @@
3535
#include "arrow/util/future.h"
3636
#include "arrow/util/iterator.h"
3737
#include "arrow/util/logging_internal.h"
38+
#include "arrow/util/key_value_metadata.h"
39+
#include "arrow/util/logging.h"
3840
#include "arrow/util/range.h"
3941
#include "arrow/util/tracing_internal.h"
4042
#include "parquet/arrow/reader.h"
4143
#include "parquet/arrow/schema.h"
4244
#include "parquet/arrow/writer.h"
4345
#include "parquet/encryption/crypto_factory.h"
4446
#include "parquet/encryption/encryption.h"
47+
#include "parquet/encryption/internal_file_decryptor.h"
4548
#include "parquet/encryption/kms_client.h"
4649
#include "parquet/file_reader.h"
4750
#include "parquet/properties.h"
@@ -1091,7 +1094,17 @@ Result<std::shared_ptr<DatasetFactory>> ParquetDatasetFactory::Make(
10911094
std::vector<std::pair<std::string, std::vector<int>>> paths_with_row_group_ids;
10921095
std::unordered_map<std::string, int> paths_to_index;
10931096

1097+
// Check if file is a metadata only file and if so extract the AADs for each RowGroup
1098+
auto kvmd = metadata->key_value_metadata();
1099+
auto encrypted_metadata_only_file =
1100+
metadata_source.path() == "_metadata" && kvmd && kvmd->Contains("row_group_aad_0");
1101+
10941102
for (int i = 0; i < metadata->num_row_groups(); i++) {
1103+
if (encrypted_metadata_only_file) {
1104+
PARQUET_ASSIGN_OR_THROW(auto aad, metadata->key_value_metadata()->Get(
1105+
"row_group_aad_" + std::to_string(i)));
1106+
metadata->set_file_decryptor_aad(aad);
1107+
}
10951108
auto row_group = metadata->RowGroup(i);
10961109
ARROW_ASSIGN_OR_RAISE(auto path,
10971110
FileFromRowGroup(filesystem.get(), base_path, *row_group,

‎cpp/src/arrow/dataset/file_parquet_encryption_test.cc‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,14 @@
3535
#include "arrow/type.h"
3636
#include "arrow/util/future.h"
3737
#include "arrow/util/thread_pool.h"
38+
#include "arrow/util/key_value_metadata.h"
3839
#include "parquet/arrow/reader.h"
40+
#include "parquet/arrow/schema.h"
3941
#include "parquet/encryption/crypto_factory.h"
4042
#include "parquet/encryption/encryption_internal.h"
4143
#include "parquet/encryption/kms_client.h"
4244
#include "parquet/encryption/test_in_memory_kms.h"
45+
#include "parquet/file_writer.h"
4346

4447
constexpr std::string_view kFooterKeyMasterKey = "0123456789012345";
4548
constexpr std::string_view kFooterKeyMasterKeyId = "footer_key";
@@ -54,6 +57,62 @@ using arrow::internal::checked_pointer_cast;
5457
namespace arrow {
5558
namespace dataset {
5659

60+
class DatasetTestBase : public ::testing::Test {
61+
public:
62+
void SetUp() override {
63+
// Creates a mock file system using the current time point.
64+
EXPECT_OK_AND_ASSIGN(file_system_, fs::internal::MockFileSystem::Make(
65+
std::chrono::system_clock::now(), {}));
66+
ASSERT_OK(file_system_->CreateDir(std::string(kBaseDir)));
67+
68+
// Init dataset and partitioning.
69+
ASSERT_NO_FATAL_FAILURE(PrepareTableAndPartitioning());
70+
71+
auto file_format = std::make_shared<ParquetFileFormat>();
72+
auto parquet_file_write_options =
73+
checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions());
74+
75+
// Write dataset.
76+
auto dataset = std::make_shared<InMemoryDataset>(table_);
77+
EXPECT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
78+
EXPECT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
79+
80+
FileSystemDatasetWriteOptions write_options;
81+
write_options.file_write_options = parquet_file_write_options;
82+
write_options.filesystem = file_system_;
83+
write_options.base_dir = kBaseDir;
84+
write_options.partitioning = partitioning_;
85+
write_options.basename_template = "part{i}.parquet";
86+
ASSERT_OK(FileSystemDataset::Write(write_options, std::move(scanner)));
87+
}
88+
89+
void PrepareTableAndPartitioning() {
90+
// Prepare table data.
91+
auto table_schema = schema({field("a", int64()), field("c", int64()),
92+
field("e", int64()), field("part", utf8())});
93+
table_ = TableFromJSON(table_schema, {R"([
94+
[ 0, 9, 1, "a" ],
95+
[ 1, 8, 2, "a" ],
96+
[ 2, 7, 1, "a" ],
97+
[ 3, 6, 2, "a" ],
98+
[ 4, 5, 1, "a" ],
99+
[ 5, 4, 2, "a" ],
100+
[ 6, 3, 1, "b" ],
101+
[ 7, 2, 2, "b" ],
102+
[ 8, 1, 1, "b" ],
103+
[ 9, 0, 2, "b" ]
104+
])"});
105+
106+
// Use a Hive-style partitioning scheme.
107+
partitioning_ = std::make_shared<HivePartitioning>(schema({field("part", utf8())}));
108+
}
109+
110+
protected:
111+
std::shared_ptr<fs::FileSystem> file_system_;
112+
std::shared_ptr<Table> table_;
113+
std::shared_ptr<Partitioning> partitioning_;
114+
};
115+
57116
struct EncryptionTestParam {
58117
bool uniform_encryption; // false is using per-column keys
59118
bool concurrently;
@@ -393,6 +452,180 @@ TEST_P(DatasetEncryptionTest, ReadSingleFile) {
393452

394453
INSTANTIATE_TEST_SUITE_P(DatasetEncryptionTest, DatasetEncryptionTest, kAllParamValues);
395454

455+
Result<std::vector<std::shared_ptr<parquet::FileMetaData>>> ReadMetadata(
456+
const std::vector<std::string>& paths, std::shared_ptr<fs::FileSystem>& file_system,
457+
const parquet::ReaderProperties& reader_properties) {
458+
if (paths.empty()) {
459+
return Status::Invalid("No files to read metadata from");
460+
}
461+
462+
std::vector<std::shared_ptr<parquet::FileMetaData>> metadata;
463+
464+
for (const auto& path : paths) {
465+
PARQUET_ASSIGN_OR_THROW(auto input, file_system->OpenInputFile(path));
466+
auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties);
467+
auto file_metadata = parquet_reader->metadata();
468+
file_metadata->set_file_path(path);
469+
metadata.push_back(file_metadata);
470+
}
471+
return metadata;
472+
}
473+
474+
// Write encrypted _metadata file and read the dataset using the metadata.
475+
TEST_F(DatasetEncryptionTest, ReadDatasetFromEncryptedMetadata) {
476+
// Create decryption properties.
477+
auto decryption_config =
478+
std::make_shared<parquet::encryption::DecryptionConfiguration>();
479+
auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
480+
parquet_decryption_config->crypto_factory = crypto_factory_;
481+
parquet_decryption_config->kms_connection_config = kms_connection_config_;
482+
parquet_decryption_config->decryption_config = std::move(decryption_config);
483+
484+
// Set scan options.
485+
auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
486+
parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config);
487+
488+
auto file_format = std::make_shared<ParquetFileFormat>();
489+
file_format->default_fragment_scan_options = std::move(parquet_scan_options);
490+
491+
auto reader_properties = parquet::default_reader_properties();
492+
decryption_config = std::make_shared<parquet::encryption::DecryptionConfiguration>();
493+
reader_properties.file_decryption_properties(
494+
crypto_factory_->GetFileDecryptionProperties(*kms_connection_config_,
495+
*decryption_config));
496+
auto encryption_config = std::make_shared<parquet::encryption::EncryptionConfiguration>(
497+
std::string(kFooterKeyName));
498+
encryption_config->column_keys = kColumnKeyMapping;
499+
500+
auto parquet_encryption_config = std::make_shared<ParquetEncryptionConfig>();
501+
// Directly assign shared_ptr objects to ParquetEncryptionConfig members
502+
parquet_encryption_config->crypto_factory = crypto_factory_;
503+
parquet_encryption_config->kms_connection_config = kms_connection_config_;
504+
parquet_encryption_config->encryption_config = encryption_config;
505+
506+
auto parquet_file_write_options =
507+
checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions());
508+
parquet_file_write_options->parquet_encryption_config =
509+
std::move(parquet_encryption_config);
510+
511+
std::vector<std::string> paths = {"part=a/part0.parquet", "part=c/part0.parquet",
512+
"part=e/part0.parquet", "part=g/part0.parquet",
513+
"part=i/part0.parquet"};
514+
515+
auto file_encryption_properties = crypto_factory_->GetFileEncryptionProperties(
516+
*kms_connection_config_, *encryption_config);
517+
518+
auto table_schema =
519+
schema({field("a", int64()), field("c", int64()), field("e", int64())});
520+
std::shared_ptr<parquet::SchemaDescriptor> schema;
521+
auto writer_props = parquet::WriterProperties::Builder().build();
522+
523+
PARQUET_ASSIGN_OR_THROW(auto metadata_list,
524+
ReadMetadata(paths, file_system_, reader_properties));
525+
PARQUET_ASSIGN_OR_THROW(auto metadata, parquet::FileMetaData::CoalesceMetadata(
526+
metadata_list, writer_props));
527+
528+
// TODO: Make sure plaintext footer mode works
529+
// encryption_config->plaintext_footer = true;
530+
file_encryption_properties = crypto_factory_->GetFileEncryptionProperties(
531+
*kms_connection_config_, *encryption_config);
532+
533+
// Write metadata to _metadata file.
534+
std::string metadata_path = "_metadata";
535+
ASSERT_OK_AND_ASSIGN(auto stream, file_system_->OpenOutputStream(metadata_path));
536+
WriteEncryptedMetadataFile(*metadata, stream, file_encryption_properties);
537+
ARROW_EXPECT_OK(stream->Close());
538+
539+
// Set scan options.
540+
decryption_config = std::make_shared<parquet::encryption::DecryptionConfiguration>();
541+
parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
542+
parquet_decryption_config->crypto_factory = crypto_factory_;
543+
parquet_decryption_config->kms_connection_config = kms_connection_config_;
544+
parquet_decryption_config->decryption_config = std::move(decryption_config);
545+
546+
parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
547+
parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config);
548+
549+
file_format = std::make_shared<ParquetFileFormat>();
550+
file_format->default_fragment_scan_options = std::move(parquet_scan_options);
551+
552+
ParquetFactoryOptions parquet_factory_options;
553+
parquet_factory_options.partitioning = partitioning_;
554+
parquet_factory_options.partition_base_dir = kBaseDir;
555+
556+
ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile(metadata_path));
557+
auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties);
558+
auto file_metadata = parquet_reader->metadata();
559+
560+
ASSERT_TRUE(file_metadata->Equals(*metadata));
561+
562+
// Create parquet dataset factory
563+
ASSERT_OK_AND_ASSIGN(auto parquet_dataset_factory,
564+
ParquetDatasetFactory::Make(metadata_path, file_system_,
565+
file_format, parquet_factory_options));
566+
567+
// Create the dataset
568+
ASSERT_OK_AND_ASSIGN(auto dataset, parquet_dataset_factory->Finish());
569+
570+
// Read dataset into table
571+
ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
572+
ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
573+
ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable());
574+
575+
// Verify the data was read correctly
576+
ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks());
577+
578+
// Validate the table
579+
ASSERT_OK(combined_table->ValidateFull());
580+
AssertTablesEqual(*table_, *combined_table);
581+
}
582+
583+
// Write _metadata file and read the dataset using the metadata.
584+
TEST_F(DatasetTestBase, ReadDatasetFromMetadata) {
585+
auto reader_properties = parquet::default_reader_properties();
586+
587+
std::vector<std::string> paths = {"part=a/part0.parquet", "part=b/part0.parquet"};
588+
std::vector<std::shared_ptr<parquet::FileMetaData>> metadata;
589+
590+
for (const auto& path : paths) {
591+
ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile(path));
592+
auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties);
593+
auto file_metadata = parquet_reader->metadata();
594+
// Make sure file_paths are stored in metadata
595+
file_metadata->set_file_path(path);
596+
metadata.push_back(file_metadata);
597+
}
598+
metadata[0]->AppendRowGroups(*metadata[1]);
599+
600+
std::string metadata_path = "_metadata";
601+
ASSERT_OK_AND_ASSIGN(auto stream, file_system_->OpenOutputStream(metadata_path));
602+
WriteMetaDataFile(*metadata[0], stream.get());
603+
ARROW_EXPECT_OK(stream->Close());
604+
605+
auto file_format = std::make_shared<ParquetFileFormat>();
606+
ParquetFactoryOptions factory_options;
607+
factory_options.partitioning = partitioning_;
608+
factory_options.partition_base_dir = kBaseDir;
609+
ASSERT_OK_AND_ASSIGN(auto dataset_factory,
610+
ParquetDatasetFactory::Make(metadata_path, file_system_,
611+
file_format, factory_options));
612+
613+
// Create the dataset
614+
ASSERT_OK_AND_ASSIGN(auto dataset, dataset_factory->Finish());
615+
616+
// Read dataset into table
617+
ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
618+
ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
619+
ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable());
620+
621+
// Verify the data was read correctly
622+
ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks());
623+
624+
// Validate the table
625+
ASSERT_OK(combined_table->ValidateFull());
626+
AssertTablesEqual(*combined_table, *table_);
627+
}
628+
396629
// GH-39444: This test covers the case where parquet dataset scanner crashes when
397630
// processing encrypted datasets over 2^15 rows in multi-threaded mode.
398631
class LargeRowCountEncryptionTest : public DatasetEncryptionTestBase {

‎cpp/src/parquet/arrow/writer.cc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -589,6 +589,14 @@ Status WriteMetaDataFile(const FileMetaData& file_metadata,
589589
return Status::OK();
590590
}
591591

592+
Status WriteEncryptedMetadataFile(
593+
const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
594+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties) {
595+
PARQUET_CATCH_NOT_OK(::parquet::WriteEncryptedMetadataFile(file_metadata, sink,
596+
file_encryption_properties));
597+
return Status::OK();
598+
}
599+
592600
Status WriteTable(const ::arrow::Table& table, ::arrow::MemoryPool* pool,
593601
std::shared_ptr<::arrow::io::OutputStream> sink, int64_t chunk_size,
594602
std::shared_ptr<WriterProperties> properties,

‎cpp/src/parquet/arrow/writer.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,12 @@ PARQUET_EXPORT
153153
::arrow::Status WriteMetaDataFile(const FileMetaData& file_metadata,
154154
::arrow::io::OutputStream* sink);
155155

156+
/// \brief Write encrypted metadata-only Parquet file to indicated Arrow OutputStream
157+
PARQUET_EXPORT
158+
::arrow::Status WriteEncryptedMetadataFile(
159+
const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
160+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties);
161+
156162
/// \brief Write a Table to Parquet.
157163
///
158164
/// This writes one table in a single shot. To write a Parquet file with

‎cpp/src/parquet/file_writer.cc‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,44 @@ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
583583
}
584584
}
585585

586+
void WriteEncryptedMetadataFile(
587+
const FileMetaData& metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
588+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties) {
589+
auto file_encryptor = std::make_unique<InternalFileEncryptor>(
590+
file_encryption_properties.get(), ::arrow::default_memory_pool());
591+
592+
if (file_encryption_properties->encrypted_footer()) {
593+
PARQUET_THROW_NOT_OK(sink->Write(kParquetEMagic, 4));
594+
595+
PARQUET_ASSIGN_OR_THROW(int64_t position, sink->Tell());
596+
auto metadata_start = static_cast<uint64_t>(position);
597+
598+
auto writer_props = parquet::WriterProperties::Builder()
599+
.encryption(file_encryption_properties)
600+
->build();
601+
auto builder = FileMetaDataBuilder::Make(metadata.schema(), writer_props);
602+
603+
auto footer_metadata = builder->Finish(metadata.key_value_metadata());
604+
auto crypto_metadata = builder->GetCryptoMetaData();
605+
606+
WriteFileCryptoMetaData(*crypto_metadata, sink.get());
607+
608+
auto footer_encryptor = file_encryptor->GetFooterEncryptor();
609+
WriteEncryptedFileMetadata(metadata, sink.get(), footer_encryptor, true);
610+
611+
PARQUET_ASSIGN_OR_THROW(position, sink->Tell());
612+
auto footer_and_crypto_len = static_cast<uint32_t>(position - metadata_start);
613+
PARQUET_THROW_NOT_OK(
614+
sink->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
615+
PARQUET_THROW_NOT_OK(sink->Write(kParquetEMagic, 4));
616+
} else {
617+
// Encrypted file with plaintext footer mode.
618+
PARQUET_THROW_NOT_OK(sink->Write(kParquetMagic, 4));
619+
auto footer_signing_encryptor = file_encryptor->GetFooterSigningEncryptor();
620+
WriteEncryptedFileMetadata(metadata, sink.get(), footer_signing_encryptor, false);
621+
}
622+
}
623+
586624
void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata,
587625
ArrowOutputStream* sink) {
588626
crypto_metadata.WriteTo(sink);

‎cpp/src/parquet/file_writer.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,11 @@ PARQUET_EXPORT
117117
void WriteMetaDataFile(const FileMetaData& file_metadata,
118118
::arrow::io::OutputStream* sink);
119119

120+
PARQUET_EXPORT
121+
void WriteEncryptedMetadataFile(
122+
const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
123+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties);
124+
120125
PARQUET_EXPORT
121126
void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
122127
ArrowOutputStream* sink,
@@ -125,9 +130,10 @@ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
125130

126131
PARQUET_EXPORT
127132
void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
128-
::arrow::io::OutputStream* sink,
133+
std::shared_ptr<::arrow::io::OutputStream> sink,
129134
const std::shared_ptr<Encryptor>& encryptor = NULLPTR,
130135
bool encrypt_footer = false);
136+
131137
PARQUET_EXPORT
132138
void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata,
133139
::arrow::io::OutputStream* sink);

0 commit comments

Comments
 (0)