Repository navigation
Commit d1dce12
committed
Initial commit
diff --git c/cpp/src/arrow/dataset/file_parquet.cc i/cpp/src/arrow/dataset/file_parquet.cc
index fd601b6..7ebf1ee 100644
--- c/cpp/src/arrow/dataset/file_parquet.cc
+++ i/cpp/src/arrow/dataset/file_parquet.cc
@@ -35,6 +35,8 @@
#include "arrow/util/future.h"
#include "arrow/util/iterator.h"
#include "arrow/util/logging_internal.h"
+#include "arrow/util/key_value_metadata.h"
+#include "arrow/util/logging.h"
#include "arrow/util/range.h"
#include "arrow/util/tracing_internal.h"
#include "parquet/arrow/reader.h"
@@ -42,6 +44,7 @@
#include "parquet/arrow/writer.h"
#include "parquet/encryption/crypto_factory.h"
#include "parquet/encryption/encryption.h"
+#include "parquet/encryption/internal_file_decryptor.h"
#include "parquet/encryption/kms_client.h"
#include "parquet/file_reader.h"
#include "parquet/properties.h"
@@ -1091,7 +1094,17 @@ Result<std::shared_ptr<DatasetFactory>> ParquetDatasetFactory::Make(
std::vector<std::pair<std::string, std::vector<int>>> paths_with_row_group_ids;
std::unordered_map<std::string, int> paths_to_index;
+ // Check if file is a metadata only file and if so extract the AADs for each RowGroup
+ auto kvmd = metadata->key_value_metadata();
+ auto encrypted_metadata_only_file =
+ metadata_source.path() == "_metadata" && kvmd && kvmd->Contains("row_group_aad_0");
+
for (int i = 0; i < metadata->num_row_groups(); i++) {
+ if (encrypted_metadata_only_file) {
+ PARQUET_ASSIGN_OR_THROW(auto aad, metadata->key_value_metadata()->Get(
+ "row_group_aad_" + std::to_string(i)));
+ metadata->set_file_decryptor_aad(aad);
+ }
auto row_group = metadata->RowGroup(i);
ARROW_ASSIGN_OR_RAISE(auto path,
FileFromRowGroup(filesystem.get(), base_path, *row_group,
diff --git c/cpp/src/arrow/dataset/file_parquet_encryption_test.cc i/cpp/src/arrow/dataset/file_parquet_encryption_test.cc
index d2e1763..0181570 100644
--- c/cpp/src/arrow/dataset/file_parquet_encryption_test.cc
+++ i/cpp/src/arrow/dataset/file_parquet_encryption_test.cc
@@ -35,11 +35,14 @@
#include "arrow/type.h"
#include "arrow/util/future.h"
#include "arrow/util/thread_pool.h"
+#include "arrow/util/key_value_metadata.h"
#include "parquet/arrow/reader.h"
+#include "parquet/arrow/schema.h"
#include "parquet/encryption/crypto_factory.h"
#include "parquet/encryption/encryption_internal.h"
#include "parquet/encryption/kms_client.h"
#include "parquet/encryption/test_in_memory_kms.h"
+#include "parquet/file_writer.h"
constexpr std::string_view kFooterKeyMasterKey = "0123456789012345";
constexpr std::string_view kFooterKeyMasterKeyId = "footer_key";
@@ -54,6 +57,62 @@ using arrow::internal::checked_pointer_cast;
namespace arrow {
namespace dataset {
+class DatasetTestBase : public ::testing::Test {
+ public:
+ void SetUp() override {
+ // Creates a mock file system using the current time point.
+ EXPECT_OK_AND_ASSIGN(file_system_, fs::internal::MockFileSystem::Make(
+ std::chrono::system_clock::now(), {}));
+ ASSERT_OK(file_system_->CreateDir(std::string(kBaseDir)));
+
+ // Init dataset and partitioning.
+ ASSERT_NO_FATAL_FAILURE(PrepareTableAndPartitioning());
+
+ auto file_format = std::make_shared<ParquetFileFormat>();
+ auto parquet_file_write_options =
+ checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions());
+
+ // Write dataset.
+ auto dataset = std::make_shared<InMemoryDataset>(table_);
+ EXPECT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
+ EXPECT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
+
+ FileSystemDatasetWriteOptions write_options;
+ write_options.file_write_options = parquet_file_write_options;
+ write_options.filesystem = file_system_;
+ write_options.base_dir = kBaseDir;
+ write_options.partitioning = partitioning_;
+ write_options.basename_template = "part{i}.parquet";
+ ASSERT_OK(FileSystemDataset::Write(write_options, std::move(scanner)));
+ }
+
+ void PrepareTableAndPartitioning() {
+ // Prepare table data.
+ auto table_schema = schema({field("a", int64()), field("c", int64()),
+ field("e", int64()), field("part", utf8())});
+ table_ = TableFromJSON(table_schema, {R"([
+ [ 0, 9, 1, "a" ],
+ [ 1, 8, 2, "a" ],
+ [ 2, 7, 1, "a" ],
+ [ 3, 6, 2, "a" ],
+ [ 4, 5, 1, "a" ],
+ [ 5, 4, 2, "a" ],
+ [ 6, 3, 1, "b" ],
+ [ 7, 2, 2, "b" ],
+ [ 8, 1, 1, "b" ],
+ [ 9, 0, 2, "b" ]
+ ])"});
+
+ // Use a Hive-style partitioning scheme.
+ partitioning_ = std::make_shared<HivePartitioning>(schema({field("part", utf8())}));
+ }
+
+ protected:
+ std::shared_ptr<fs::FileSystem> file_system_;
+ std::shared_ptr<Table> table_;
+ std::shared_ptr<Partitioning> partitioning_;
+};
+
struct EncryptionTestParam {
bool uniform_encryption; // false is using per-column keys
bool concurrently;
@@ -393,6 +452,180 @@ TEST_P(DatasetEncryptionTest, ReadSingleFile) {
INSTANTIATE_TEST_SUITE_P(DatasetEncryptionTest, DatasetEncryptionTest, kAllParamValues);
+Result<std::vector<std::shared_ptr<parquet::FileMetaData>>> ReadMetadata(
+ const std::vector<std::string>& paths, std::shared_ptr<fs::FileSystem>& file_system,
+ const parquet::ReaderProperties& reader_properties) {
+ if (paths.empty()) {
+ return Status::Invalid("No files to read metadata from");
+ }
+
+ std::vector<std::shared_ptr<parquet::FileMetaData>> metadata;
+
+ for (const auto& path : paths) {
+ PARQUET_ASSIGN_OR_THROW(auto input, file_system->OpenInputFile(path));
+ auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties);
+ auto file_metadata = parquet_reader->metadata();
+ file_metadata->set_file_path(path);
+ metadata.push_back(file_metadata);
+ }
+ return metadata;
+}
+
+// Write encrypted _metadata file and read the dataset using the metadata.
+TEST_F(DatasetEncryptionTest, ReadDatasetFromEncryptedMetadata) {
+ // Create decryption properties.
+ auto decryption_config =
+ std::make_shared<parquet::encryption::DecryptionConfiguration>();
+ auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
+ parquet_decryption_config->crypto_factory = crypto_factory_;
+ parquet_decryption_config->kms_connection_config = kms_connection_config_;
+ parquet_decryption_config->decryption_config = std::move(decryption_config);
+
+ // Set scan options.
+ auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
+ parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config);
+
+ auto file_format = std::make_shared<ParquetFileFormat>();
+ file_format->default_fragment_scan_options = std::move(parquet_scan_options);
+
+ auto reader_properties = parquet::default_reader_properties();
+ decryption_config = std::make_shared<parquet::encryption::DecryptionConfiguration>();
+ reader_properties.file_decryption_properties(
+ crypto_factory_->GetFileDecryptionProperties(*kms_connection_config_,
+ *decryption_config));
+ auto encryption_config = std::make_shared<parquet::encryption::EncryptionConfiguration>(
+ std::string(kFooterKeyName));
+ encryption_config->column_keys = kColumnKeyMapping;
+
+ auto parquet_encryption_config = std::make_shared<ParquetEncryptionConfig>();
+ // Directly assign shared_ptr objects to ParquetEncryptionConfig members
+ parquet_encryption_config->crypto_factory = crypto_factory_;
+ parquet_encryption_config->kms_connection_config = kms_connection_config_;
+ parquet_encryption_config->encryption_config = encryption_config;
+
+ auto parquet_file_write_options =
+ checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions());
+ parquet_file_write_options->parquet_encryption_config =
+ std::move(parquet_encryption_config);
+
+ std::vector<std::string> paths = {"part=a/part0.parquet", "part=c/part0.parquet",
+ "part=e/part0.parquet", "part=g/part0.parquet",
+ "part=i/part0.parquet"};
+
+ auto file_encryption_properties = crypto_factory_->GetFileEncryptionProperties(
+ *kms_connection_config_, *encryption_config);
+
+ auto table_schema =
+ schema({field("a", int64()), field("c", int64()), field("e", int64())});
+ std::shared_ptr<parquet::SchemaDescriptor> schema;
+ auto writer_props = parquet::WriterProperties::Builder().build();
+
+ PARQUET_ASSIGN_OR_THROW(auto metadata_list,
+ ReadMetadata(paths, file_system_, reader_properties));
+ PARQUET_ASSIGN_OR_THROW(auto metadata, parquet::FileMetaData::CoalesceMetadata(
+ metadata_list, writer_props));
+
+ // TODO: Make sure plaintext footer mode works
+ // encryption_config->plaintext_footer = true;
+ file_encryption_properties = crypto_factory_->GetFileEncryptionProperties(
+ *kms_connection_config_, *encryption_config);
+
+ // Write metadata to _metadata file.
+ std::string metadata_path = "_metadata";
+ ASSERT_OK_AND_ASSIGN(auto stream, file_system_->OpenOutputStream(metadata_path));
+ WriteEncryptedMetadataFile(*metadata, stream, file_encryption_properties);
+ ARROW_EXPECT_OK(stream->Close());
+
+ // Set scan options.
+ decryption_config = std::make_shared<parquet::encryption::DecryptionConfiguration>();
+ parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
+ parquet_decryption_config->crypto_factory = crypto_factory_;
+ parquet_decryption_config->kms_connection_config = kms_connection_config_;
+ parquet_decryption_config->decryption_config = std::move(decryption_config);
+
+ parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
+ parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config);
+
+ file_format = std::make_shared<ParquetFileFormat>();
+ file_format->default_fragment_scan_options = std::move(parquet_scan_options);
+
+ ParquetFactoryOptions parquet_factory_options;
+ parquet_factory_options.partitioning = partitioning_;
+ parquet_factory_options.partition_base_dir = kBaseDir;
+
+ ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile(metadata_path));
+ auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties);
+ auto file_metadata = parquet_reader->metadata();
+
+ ASSERT_TRUE(file_metadata->Equals(*metadata));
+
+ // Create parquet dataset factory
+ ASSERT_OK_AND_ASSIGN(auto parquet_dataset_factory,
+ ParquetDatasetFactory::Make(metadata_path, file_system_,
+ file_format, parquet_factory_options));
+
+ // Create the dataset
+ ASSERT_OK_AND_ASSIGN(auto dataset, parquet_dataset_factory->Finish());
+
+ // Read dataset into table
+ ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
+ ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
+ ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable());
+
+ // Verify the data was read correctly
+ ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks());
+
+ // Validate the table
+ ASSERT_OK(combined_table->ValidateFull());
+ AssertTablesEqual(*table_, *combined_table);
+}
+
+// Write _metadata file and read the dataset using the metadata.
+TEST_F(DatasetTestBase, ReadDatasetFromMetadata) {
+ auto reader_properties = parquet::default_reader_properties();
+
+ std::vector<std::string> paths = {"part=a/part0.parquet", "part=b/part0.parquet"};
+ std::vector<std::shared_ptr<parquet::FileMetaData>> metadata;
+
+ for (const auto& path : paths) {
+ ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile(path));
+ auto parquet_reader = parquet::ParquetFileReader::Open(input, reader_properties);
+ auto file_metadata = parquet_reader->metadata();
+ // Make sure file_paths are stored in metadata
+ file_metadata->set_file_path(path);
+ metadata.push_back(file_metadata);
+ }
+ metadata[0]->AppendRowGroups(*metadata[1]);
+
+ std::string metadata_path = "_metadata";
+ ASSERT_OK_AND_ASSIGN(auto stream, file_system_->OpenOutputStream(metadata_path));
+ WriteMetaDataFile(*metadata[0], stream.get());
+ ARROW_EXPECT_OK(stream->Close());
+
+ auto file_format = std::make_shared<ParquetFileFormat>();
+ ParquetFactoryOptions factory_options;
+ factory_options.partitioning = partitioning_;
+ factory_options.partition_base_dir = kBaseDir;
+ ASSERT_OK_AND_ASSIGN(auto dataset_factory,
+ ParquetDatasetFactory::Make(metadata_path, file_system_,
+ file_format, factory_options));
+
+ // Create the dataset
+ ASSERT_OK_AND_ASSIGN(auto dataset, dataset_factory->Finish());
+
+ // Read dataset into table
+ ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
+ ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
+ ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable());
+
+ // Verify the data was read correctly
+ ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks());
+
+ // Validate the table
+ ASSERT_OK(combined_table->ValidateFull());
+ AssertTablesEqual(*combined_table, *table_);
+}
+
// apacheGH-39444: This test covers the case where parquet dataset scanner crashes when
// processing encrypted datasets over 2^15 rows in multi-threaded mode.
class LargeRowCountEncryptionTest : public DatasetEncryptionTestBase {
diff --git c/cpp/src/parquet/arrow/writer.cc i/cpp/src/parquet/arrow/writer.cc
index 5307465..43ac9a6 100644
--- c/cpp/src/parquet/arrow/writer.cc
+++ i/cpp/src/parquet/arrow/writer.cc
@@ -589,6 +589,14 @@ Status WriteMetaDataFile(const FileMetaData& file_metadata,
return Status::OK();
}
+Status WriteEncryptedMetadataFile(
+ const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties) {
+ PARQUET_CATCH_NOT_OK(::parquet::WriteEncryptedMetadataFile(file_metadata, sink,
+ file_encryption_properties));
+ return Status::OK();
+}
+
Status WriteTable(const ::arrow::Table& table, ::arrow::MemoryPool* pool,
std::shared_ptr<::arrow::io::OutputStream> sink, int64_t chunk_size,
std::shared_ptr<WriterProperties> properties,
diff --git c/cpp/src/parquet/arrow/writer.h i/cpp/src/parquet/arrow/writer.h
index 8ec8796..194d91f 100644
--- c/cpp/src/parquet/arrow/writer.h
+++ i/cpp/src/parquet/arrow/writer.h
@@ -153,6 +153,12 @@ PARQUET_EXPORT
::arrow::Status WriteMetaDataFile(const FileMetaData& file_metadata,
::arrow::io::OutputStream* sink);
+/// \brief Write encrypted metadata-only Parquet file to indicated Arrow OutputStream
+PARQUET_EXPORT
+::arrow::Status WriteEncryptedMetadataFile(
+ const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties);
+
/// \brief Write a Table to Parquet.
///
/// This writes one table in a single shot. To write a Parquet file with
diff --git c/cpp/src/parquet/file_writer.cc i/cpp/src/parquet/file_writer.cc
index 498fda4..3a35fbd 100644
--- c/cpp/src/parquet/file_writer.cc
+++ i/cpp/src/parquet/file_writer.cc
@@ -583,6 +583,46 @@ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
}
}
+void WriteEncryptedMetadataFile(
+ const FileMetaData& metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties) {
+ auto file_encryptor = std::make_unique<InternalFileEncryptor>(
+ file_encryption_properties.get(), ::arrow::default_memory_pool());
+
+ if (file_encryption_properties->encrypted_footer()) {
+ PARQUET_THROW_NOT_OK(sink->Write(kParquetEMagic, 4));
+
+ PARQUET_ASSIGN_OR_THROW(int64_t position, sink->Tell());
+ auto metadata_start = static_cast<uint64_t>(position);
+
+ auto writer_props = parquet::WriterProperties::Builder()
+ .encryption(file_encryption_properties)
+ ->build();
+ auto builder = FileMetaDataBuilder::Make(metadata.schema(), writer_props);
+
+ auto footer_metadata = builder->Finish(metadata.key_value_metadata());
+ auto crypto_metadata = builder->GetCryptoMetaData();
+
+ WriteFileCryptoMetaData(*crypto_metadata, sink.get());
+
+ auto footer_encryptor = file_encryptor->GetFooterEncryptor();
+ WriteEncryptedFileMetadata(metadata, sink.get(), footer_encryptor, true);
+
+ PARQUET_ASSIGN_OR_THROW(position, sink->Tell());
+ auto footer_and_crypto_len = static_cast<uint32_t>(position - metadata_start);
+ PARQUET_THROW_NOT_OK(
+ sink->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
+ PARQUET_THROW_NOT_OK(sink->Write(kParquetEMagic, 4));
+ } else {
+ // Encrypted file with plaintext footer mode.
+ PARQUET_THROW_NOT_OK(sink->Write(kParquetMagic, 4));
+ auto footer_signing_encryptor = file_encryptor->GetFooterSigningEncryptor();
+ WriteEncryptedFileMetadata(metadata, sink.get(), footer_signing_encryptor, false);
+ }
+
+ file_encryptor->WipeOutEncryptionKeys();
+}
+
void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata,
ArrowOutputStream* sink) {
crypto_metadata.WriteTo(sink);
diff --git c/cpp/src/parquet/file_writer.h i/cpp/src/parquet/file_writer.h
index d5ea1d7..60b8474 100644
--- c/cpp/src/parquet/file_writer.h
+++ i/cpp/src/parquet/file_writer.h
@@ -117,6 +117,11 @@ PARQUET_EXPORT
void WriteMetaDataFile(const FileMetaData& file_metadata,
::arrow::io::OutputStream* sink);
+PARQUET_EXPORT
+void WriteEncryptedMetadataFile(
+ const FileMetaData& file_metadata, std::shared_ptr<::arrow::io::OutputStream> sink,
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties);
+
PARQUET_EXPORT
void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
ArrowOutputStream* sink,
@@ -125,9 +130,10 @@ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
PARQUET_EXPORT
void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
- ::arrow::io::OutputStream* sink,
+ std::shared_ptr<::arrow::io::OutputStream> sink,
const std::shared_ptr<Encryptor>& encryptor = NULLPTR,
bool encrypt_footer = false);
+
PARQUET_EXPORT
void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata,
::arrow::io::OutputStream* sink);
diff --git c/cpp/src/parquet/metadata.cc i/cpp/src/parquet/metadata.cc
index 9fbf8f1..ea8e4f1 100644
--- c/cpp/src/parquet/metadata.cc
+++ i/cpp/src/parquet/metadata.cc
@@ -905,6 +905,13 @@ class FileMetaData::FileMetaDataImpl {
out->impl_->key_value_metadata_ = key_value_metadata_;
out->impl_->file_decryptor_ = file_decryptor_;
+ if (key_value_metadata_ && key_value_metadata_->Contains("row_group_aad_0")) {
+ PARQUET_ASSIGN_OR_THROW(
+ auto aad,
+ key_value_metadata_->Get("row_group_aad_" + std::to_string(row_groups[0])));
+ out->set_file_decryptor_aad(aad);
+ }
+
return out;
}
@@ -1029,6 +1036,10 @@ EncryptionAlgorithm FileMetaData::encryption_algorithm() const {
return impl_->encryption_algorithm();
}
+const std::string& FileMetaData::file_aad() const {
+ return impl_->file_decryptor()->file_aad();
+}
+
const std::string& FileMetaData::footer_signing_key_metadata() const {
return impl_->footer_signing_key_metadata();
}
@@ -1038,6 +1049,13 @@ void FileMetaData::set_file_decryptor(
impl_->set_file_decryptor(std::move(file_decryptor));
}
+void FileMetaData::set_file_decryptor_aad(const std::string& aad) {
+ auto fd = impl_->file_decryptor();
+ auto file_decryptor = std::make_shared<parquet::InternalFileDecryptor>(
+ fd->properties(), aad, fd->algorithm(), fd->footer_key_metadata(), fd->pool());
+ set_file_decryptor(file_decryptor);
+}
+
const std::shared_ptr<InternalFileDecryptor>& FileMetaData::file_decryptor() const {
return impl_->file_decryptor();
}
@@ -1089,6 +1107,36 @@ void FileMetaData::WriteTo(::arrow::io::OutputStream* dst,
return impl_->WriteTo(dst, encryptor);
}
+::arrow::Result<std::shared_ptr<parquet::FileMetaData>> FileMetaData::CoalesceMetadata(
+ std::vector<std::shared_ptr<parquet::FileMetaData>>& metadata_list,
+ std::shared_ptr<parquet::WriterProperties>& writer_props) {
+ if (metadata_list.empty()) {
+ return ::arrow::Status::Invalid("No metadata to coalesce");
+ }
+
+ std::vector<std::string> values, keys;
+
+ // Read metadata from all dataset files and store AADs.
+ for (size_t i = 0; i < metadata_list.size(); i++) {
+ const auto& file_metadata = metadata_list[i];
+ keys.push_back("row_group_aad_" + std::to_string(i));
+ values.push_back(file_metadata->file_aad());
+ if (i > 0) {
+ metadata_list[0]->AppendRowGroups(*file_metadata);
+ }
+ }
+
+ // Create a new FileMetadata object with the created AADs as key_value_metadata.
+ auto fmd_builder =
+ parquet::FileMetaDataBuilder::Make(metadata_list[0]->schema(), writer_props);
+ const std::shared_ptr<const KeyValueMetadata> file_aad_metadata =
+ ::arrow::key_value_metadata(keys, values);
+ auto metadata = fmd_builder->Finish(file_aad_metadata);
+ metadata->AppendRowGroups(*metadata_list[0]);
+
+ return metadata;
+}
+
class FileCryptoMetaData::FileCryptoMetaDataImpl {
public:
FileCryptoMetaDataImpl() = default;
diff --git c/cpp/src/parquet/metadata.h i/cpp/src/parquet/metadata.h
index 9a3964f..378c1f3 100644
--- c/cpp/src/parquet/metadata.h
+++ i/cpp/src/parquet/metadata.h
@@ -328,6 +328,8 @@ class PARQUET_EXPORT FileMetaData {
EncryptionAlgorithm encryption_algorithm() const;
const std::string& footer_signing_key_metadata() const;
+ const std::string& file_aad() const;
+
/// \brief Verify signature of FileMetaData when file is encrypted but footer
/// is not encrypted (plaintext footer).
bool VerifySignature(const void* signature);
@@ -377,6 +379,23 @@ class PARQUET_EXPORT FileMetaData {
/// debug text (if true).
std::string SerializeUnencrypted(bool scrub, bool debug) const;
+ /// \brief Coalesce metadata from multiple files into a single metadata file by
+ /// appending row groups.
+ ///
+ /// \param[in] metadata_list list of FileMetaData objects to coalesce.
+ /// \param[in] writer_props WriterProperties to use to create coalesced FileMetaData
+ /// object.
+ /// \return a new FileMetaData object containing all row groups from the input
+ /// FileMetaData objects.
+ static ::arrow::Result<std::shared_ptr<FileMetaData>> CoalesceMetadata(
+ std::vector<std::shared_ptr<FileMetaData>>& metadata_list,
+ std::shared_ptr<WriterProperties>& writer_props);
+
+ /// \brief Set the AAD of decryptor of the file.
+ ///
+ /// \param[in] aad AAD to set on the decryptor.
+ void set_file_decryptor_aad(const std::string& aad);
+
private:
friend FileMetaDataBuilder;
friend class SerializedFile;1 parent 586ed92 commit d1dce12
8 files changed
Lines changed: 372 additions & 1 deletion
File tree
- cpp/src
- arrow/dataset
- parquet
- arrow
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
35 | 35 | | |
36 | 36 | | |
37 | 37 | | |
| 38 | + | |
| 39 | + | |
38 | 40 | | |
39 | 41 | | |
40 | 42 | | |
41 | 43 | | |
42 | 44 | | |
43 | 45 | | |
44 | 46 | | |
| 47 | + | |
45 | 48 | | |
46 | 49 | | |
47 | 50 | | |
| |||
1091 | 1094 | | |
1092 | 1095 | | |
1093 | 1096 | | |
| 1097 | + | |
| 1098 | + | |
| 1099 | + | |
| 1100 | + | |
| 1101 | + | |
1094 | 1102 | | |
| 1103 | + | |
| 1104 | + | |
| 1105 | + | |
| 1106 | + | |
| 1107 | + | |
1095 | 1108 | | |
1096 | 1109 | | |
1097 | 1110 | | |
| |||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
35 | 35 | | |
36 | 36 | | |
37 | 37 | | |
| 38 | + | |
38 | 39 | | |
| 40 | + | |
39 | 41 | | |
40 | 42 | | |
41 | 43 | | |
42 | 44 | | |
| 45 | + | |
43 | 46 | | |
44 | 47 | | |
45 | 48 | | |
| |||
54 | 57 | | |
55 | 58 | | |
56 | 59 | | |
| 60 | + | |
| 61 | + | |
| 62 | + | |
| 63 | + | |
| 64 | + | |
| 65 | + | |
| 66 | + | |
| 67 | + | |
| 68 | + | |
| 69 | + | |
| 70 | + | |
| 71 | + | |
| 72 | + | |
| 73 | + | |
| 74 | + | |
| 75 | + | |
| 76 | + | |
| 77 | + | |
| 78 | + | |
| 79 | + | |
| 80 | + | |
| 81 | + | |
| 82 | + | |
| 83 | + | |
| 84 | + | |
| 85 | + | |
| 86 | + | |
| 87 | + | |
| 88 | + | |
| 89 | + | |
| 90 | + | |
| 91 | + | |
| 92 | + | |
| 93 | + | |
| 94 | + | |
| 95 | + | |
| 96 | + | |
| 97 | + | |
| 98 | + | |
| 99 | + | |
| 100 | + | |
| 101 | + | |
| 102 | + | |
| 103 | + | |
| 104 | + | |
| 105 | + | |
| 106 | + | |
| 107 | + | |
| 108 | + | |
| 109 | + | |
| 110 | + | |
| 111 | + | |
| 112 | + | |
| 113 | + | |
| 114 | + | |
| 115 | + | |
57 | 116 | | |
58 | 117 | | |
59 | 118 | | |
| |||
393 | 452 | | |
394 | 453 | | |
395 | 454 | | |
| 455 | + | |
| 456 | + | |
| 457 | + | |
| 458 | + | |
| 459 | + | |
| 460 | + | |
| 461 | + | |
| 462 | + | |
| 463 | + | |
| 464 | + | |
| 465 | + | |
| 466 | + | |
| 467 | + | |
| 468 | + | |
| 469 | + | |
| 470 | + | |
| 471 | + | |
| 472 | + | |
| 473 | + | |
| 474 | + | |
| 475 | + | |
| 476 | + | |
| 477 | + | |
| 478 | + | |
| 479 | + | |
| 480 | + | |
| 481 | + | |
| 482 | + | |
| 483 | + | |
| 484 | + | |
| 485 | + | |
| 486 | + | |
| 487 | + | |
| 488 | + | |
| 489 | + | |
| 490 | + | |
| 491 | + | |
| 492 | + | |
| 493 | + | |
| 494 | + | |
| 495 | + | |
| 496 | + | |
| 497 | + | |
| 498 | + | |
| 499 | + | |
| 500 | + | |
| 501 | + | |
| 502 | + | |
| 503 | + | |
| 504 | + | |
| 505 | + | |
| 506 | + | |
| 507 | + | |
| 508 | + | |
| 509 | + | |
| 510 | + | |
| 511 | + | |
| 512 | + | |
| 513 | + | |
| 514 | + | |
| 515 | + | |
| 516 | + | |
| 517 | + | |
| 518 | + | |
| 519 | + | |
| 520 | + | |
| 521 | + | |
| 522 | + | |
| 523 | + | |
| 524 | + | |
| 525 | + | |
| 526 | + | |
| 527 | + | |
| 528 | + | |
| 529 | + | |
| 530 | + | |
| 531 | + | |
| 532 | + | |
| 533 | + | |
| 534 | + | |
| 535 | + | |
| 536 | + | |
| 537 | + | |
| 538 | + | |
| 539 | + | |
| 540 | + | |
| 541 | + | |
| 542 | + | |
| 543 | + | |
| 544 | + | |
| 545 | + | |
| 546 | + | |
| 547 | + | |
| 548 | + | |
| 549 | + | |
| 550 | + | |
| 551 | + | |
| 552 | + | |
| 553 | + | |
| 554 | + | |
| 555 | + | |
| 556 | + | |
| 557 | + | |
| 558 | + | |
| 559 | + | |
| 560 | + | |
| 561 | + | |
| 562 | + | |
| 563 | + | |
| 564 | + | |
| 565 | + | |
| 566 | + | |
| 567 | + | |
| 568 | + | |
| 569 | + | |
| 570 | + | |
| 571 | + | |
| 572 | + | |
| 573 | + | |
| 574 | + | |
| 575 | + | |
| 576 | + | |
| 577 | + | |
| 578 | + | |
| 579 | + | |
| 580 | + | |
| 581 | + | |
| 582 | + | |
| 583 | + | |
| 584 | + | |
| 585 | + | |
| 586 | + | |
| 587 | + | |
| 588 | + | |
| 589 | + | |
| 590 | + | |
| 591 | + | |
| 592 | + | |
| 593 | + | |
| 594 | + | |
| 595 | + | |
| 596 | + | |
| 597 | + | |
| 598 | + | |
| 599 | + | |
| 600 | + | |
| 601 | + | |
| 602 | + | |
| 603 | + | |
| 604 | + | |
| 605 | + | |
| 606 | + | |
| 607 | + | |
| 608 | + | |
| 609 | + | |
| 610 | + | |
| 611 | + | |
| 612 | + | |
| 613 | + | |
| 614 | + | |
| 615 | + | |
| 616 | + | |
| 617 | + | |
| 618 | + | |
| 619 | + | |
| 620 | + | |
| 621 | + | |
| 622 | + | |
| 623 | + | |
| 624 | + | |
| 625 | + | |
| 626 | + | |
| 627 | + | |
| 628 | + | |
396 | 629 | | |
397 | 630 | | |
398 | 631 | | |
| |||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
589 | 589 | | |
590 | 590 | | |
591 | 591 | | |
| 592 | + | |
| 593 | + | |
| 594 | + | |
| 595 | + | |
| 596 | + | |
| 597 | + | |
| 598 | + | |
| 599 | + | |
592 | 600 | | |
593 | 601 | | |
594 | 602 | | |
| |||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
153 | 153 | | |
154 | 154 | | |
155 | 155 | | |
| 156 | + | |
| 157 | + | |
| 158 | + | |
| 159 | + | |
| 160 | + | |
| 161 | + | |
156 | 162 | | |
157 | 163 | | |
158 | 164 | | |
| |||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
583 | 583 | | |
584 | 584 | | |
585 | 585 | | |
| 586 | + | |
| 587 | + | |
| 588 | + | |
| 589 | + | |
| 590 | + | |
| 591 | + | |
| 592 | + | |
| 593 | + | |
| 594 | + | |
| 595 | + | |
| 596 | + | |
| 597 | + | |
| 598 | + | |
| 599 | + | |
| 600 | + | |
| 601 | + | |
| 602 | + | |
| 603 | + | |
| 604 | + | |
| 605 | + | |
| 606 | + | |
| 607 | + | |
| 608 | + | |
| 609 | + | |
| 610 | + | |
| 611 | + | |
| 612 | + | |
| 613 | + | |
| 614 | + | |
| 615 | + | |
| 616 | + | |
| 617 | + | |
| 618 | + | |
| 619 | + | |
| 620 | + | |
| 621 | + | |
| 622 | + | |
| 623 | + | |
586 | 624 | | |
587 | 625 | | |
588 | 626 | | |
| |||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
117 | 117 | | |
118 | 118 | | |
119 | 119 | | |
| 120 | + | |
| 121 | + | |
| 122 | + | |
| 123 | + | |
| 124 | + | |
120 | 125 | | |
121 | 126 | | |
122 | 127 | | |
| |||
125 | 130 | | |
126 | 131 | | |
127 | 132 | | |
128 | | - | |
| 133 | + | |
129 | 134 | | |
130 | 135 | | |
| 136 | + | |
131 | 137 | | |
132 | 138 | | |
133 | 139 | | |
| |||
0 commit comments