|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +#include <string_view> |
| 19 | + |
| 20 | +#include "gtest/gtest.h" |
| 21 | + |
| 22 | +#include <arrow/dataset/dataset.h> |
| 23 | +#include <arrow/dataset/file_base.h> |
| 24 | +#include <arrow/dataset/file_parquet.h> |
| 25 | +#include "arrow/array.h" |
| 26 | +#include "arrow/dataset/parquet_encryption_config.h" |
| 27 | +#include "arrow/dataset/partition.h" |
| 28 | +#include "arrow/filesystem/mockfs.h" |
| 29 | +#include "arrow/io/api.h" |
| 30 | +#include "arrow/status.h" |
| 31 | +#include "arrow/table.h" |
| 32 | +#include "arrow/testing/gtest_util.h" |
| 33 | +#include "arrow/type.h" |
| 34 | +#include "parquet/arrow/reader.h" |
| 35 | +#include "parquet/encryption/crypto_factory.h" |
| 36 | +#include "parquet/encryption/encryption.h" |
| 37 | +#include "parquet/encryption/kms_client.h" |
| 38 | +#include "parquet/encryption/test_in_memory_kms.h" |
| 39 | + |
| 40 | +constexpr std::string_view kFooterKeyMasterKey = "0123456789012345"; |
| 41 | +constexpr std::string_view kFooterKeyMasterKeyId = "footer_key"; |
| 42 | +constexpr std::string_view kFooterKeyName = "footer_key"; |
| 43 | +constexpr std::string_view kColumnMasterKey = "1234567890123450"; |
| 44 | +constexpr std::string_view kColumnMasterKeyId = "col_key"; |
| 45 | +constexpr std::string_view kColumnKeyMapping = "col_key: a"; |
| 46 | +constexpr std::string_view kBaseDir = ""; |
| 47 | + |
| 48 | +using arrow::internal::checked_pointer_cast; |
| 49 | + |
| 50 | +namespace arrow { |
| 51 | +namespace dataset { |
| 52 | + |
| 53 | +class DatasetEncryptionTest : public ::testing::Test { |
| 54 | + public: |
| 55 | + // This function creates a mock file system using the current time point, creates a |
| 56 | + // directory with the given base directory path, and writes a dataset to it using |
| 57 | + // provided Parquet file write options. The dataset is partitioned using a Hive |
| 58 | + // partitioning scheme. The function also checks if the written files exist in the file |
| 59 | + // system. |
| 60 | + static void SetUpTestSuite() { |
| 61 | + // Creates a mock file system using the current time point. |
| 62 | + EXPECT_OK_AND_ASSIGN(file_system_, fs::internal::MockFileSystem::Make( |
| 63 | + std::chrono::system_clock::now(), {})); |
| 64 | + ASSERT_OK(file_system_->CreateDir(std::string(kBaseDir))); |
| 65 | + |
| 66 | + // Prepare table data. |
| 67 | + auto table_schema = schema({field("a", int64()), field("c", int64()), |
| 68 | + field("e", int64()), field("part", utf8())}); |
| 69 | + table_ = TableFromJSON(table_schema, {R"([ |
| 70 | + [ 0, 9, 1, "a" ], |
| 71 | + [ 1, 8, 2, "a" ], |
| 72 | + [ 2, 7, 1, "c" ], |
| 73 | + [ 3, 6, 2, "c" ], |
| 74 | + [ 4, 5, 1, "e" ], |
| 75 | + [ 5, 4, 2, "e" ], |
| 76 | + [ 6, 3, 1, "g" ], |
| 77 | + [ 7, 2, 2, "g" ], |
| 78 | + [ 8, 1, 1, "i" ], |
| 79 | + [ 9, 0, 2, "i" ] |
| 80 | + ])"}); |
| 81 | + |
| 82 | + // Use a Hive-style partitioning scheme. |
| 83 | + partitioning_ = std::make_shared<HivePartitioning>(schema({field("part", utf8())})); |
| 84 | + |
| 85 | + // Prepare encryption properties. |
| 86 | + std::unordered_map<std::string, std::string> key_map; |
| 87 | + key_map.emplace(kColumnMasterKeyId, kColumnMasterKey); |
| 88 | + key_map.emplace(kFooterKeyMasterKeyId, kFooterKeyMasterKey); |
| 89 | + |
| 90 | + crypto_factory_ = std::make_shared<parquet::encryption::CryptoFactory>(); |
| 91 | + auto kms_client_factory = |
| 92 | + std::make_shared<parquet::encryption::TestOnlyInMemoryKmsClientFactory>( |
| 93 | + /*wrap_locally=*/true, key_map); |
| 94 | + crypto_factory_->RegisterKmsClientFactory(std::move(kms_client_factory)); |
| 95 | + kms_connection_config_ = std::make_shared<parquet::encryption::KmsConnectionConfig>(); |
| 96 | + |
| 97 | + // Set write options with encryption configuration. |
| 98 | + auto encryption_config = |
| 99 | + std::make_shared<parquet::encryption::EncryptionConfiguration>( |
| 100 | + std::string(kFooterKeyName)); |
| 101 | + encryption_config->column_keys = kColumnKeyMapping; |
| 102 | + auto parquet_encryption_config = std::make_shared<ParquetEncryptionConfig>(); |
| 103 | + // Directly assign shared_ptr objects to ParquetEncryptionConfig members |
| 104 | + parquet_encryption_config->crypto_factory = crypto_factory_; |
| 105 | + parquet_encryption_config->kms_connection_config = kms_connection_config_; |
| 106 | + parquet_encryption_config->encryption_config = std::move(encryption_config); |
| 107 | + |
| 108 | + auto file_format = std::make_shared<ParquetFileFormat>(); |
| 109 | + auto parquet_file_write_options = |
| 110 | + checked_pointer_cast<ParquetFileWriteOptions>(file_format->DefaultWriteOptions()); |
| 111 | + parquet_file_write_options->parquet_encryption_config = |
| 112 | + std::move(parquet_encryption_config); |
| 113 | + |
| 114 | + // Write dataset. |
| 115 | + auto dataset = std::make_shared<InMemoryDataset>(table_); |
| 116 | + EXPECT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan()); |
| 117 | + EXPECT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish()); |
| 118 | + |
| 119 | + FileSystemDatasetWriteOptions write_options; |
| 120 | + write_options.file_write_options = parquet_file_write_options; |
| 121 | + write_options.filesystem = file_system_; |
| 122 | + write_options.base_dir = kBaseDir; |
| 123 | + write_options.partitioning = partitioning_; |
| 124 | + write_options.basename_template = "part{i}.parquet"; |
| 125 | + ASSERT_OK(FileSystemDataset::Write(write_options, std::move(scanner))); |
| 126 | + } |
| 127 | + |
| 128 | + protected: |
| 129 | + inline static std::shared_ptr<fs::FileSystem> file_system_; |
| 130 | + inline static std::shared_ptr<Table> table_; |
| 131 | + inline static std::shared_ptr<HivePartitioning> partitioning_; |
| 132 | + inline static std::shared_ptr<parquet::encryption::CryptoFactory> crypto_factory_; |
| 133 | + inline static std::shared_ptr<parquet::encryption::KmsConnectionConfig> |
| 134 | + kms_connection_config_; |
| 135 | +}; |
| 136 | + |
| 137 | +// This test demonstrates the process of writing a partitioned Parquet file with the same |
| 138 | +// encryption properties applied to each file within the dataset. The encryption |
| 139 | +// properties are determined based on the selected columns. After writing the dataset, the |
| 140 | +// test reads the data back and verifies that it can be successfully decrypted and |
| 141 | +// scanned. |
| 142 | +TEST_F(DatasetEncryptionTest, WriteReadDatasetWithEncryption) { |
| 143 | + // Create decryption properties. |
| 144 | + auto decryption_config = |
| 145 | + std::make_shared<parquet::encryption::DecryptionConfiguration>(); |
| 146 | + auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>(); |
| 147 | + parquet_decryption_config->crypto_factory = crypto_factory_; |
| 148 | + parquet_decryption_config->kms_connection_config = kms_connection_config_; |
| 149 | + parquet_decryption_config->decryption_config = std::move(decryption_config); |
| 150 | + |
| 151 | + // Set scan options. |
| 152 | + auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>(); |
| 153 | + parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config); |
| 154 | + |
| 155 | + auto file_format = std::make_shared<ParquetFileFormat>(); |
| 156 | + file_format->default_fragment_scan_options = std::move(parquet_scan_options); |
| 157 | + |
| 158 | + // Get FileInfo objects for all files under the base directory |
| 159 | + fs::FileSelector selector; |
| 160 | + selector.base_dir = kBaseDir; |
| 161 | + selector.recursive = true; |
| 162 | + |
| 163 | + FileSystemFactoryOptions factory_options; |
| 164 | + factory_options.partitioning = partitioning_; |
| 165 | + factory_options.partition_base_dir = kBaseDir; |
| 166 | + ASSERT_OK_AND_ASSIGN(auto dataset_factory, |
| 167 | + FileSystemDatasetFactory::Make(file_system_, selector, file_format, |
| 168 | + factory_options)); |
| 169 | + |
| 170 | + // Read dataset into table |
| 171 | + ASSERT_OK_AND_ASSIGN(auto dataset, dataset_factory->Finish()); |
| 172 | + ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan()); |
| 173 | + ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish()); |
| 174 | + ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable()); |
| 175 | + |
| 176 | + // Verify the data was read correctly |
| 177 | + ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks()); |
| 178 | + // Validate the table |
| 179 | + ASSERT_OK(combined_table->ValidateFull()); |
| 180 | + AssertTablesEqual(*combined_table, *table_); |
| 181 | +} |
| 182 | + |
| 183 | +// Read a single parquet file with and without decryption properties. |
| 184 | +TEST_F(DatasetEncryptionTest, ReadSingleFile) { |
| 185 | + // Open the Parquet file. |
| 186 | + ASSERT_OK_AND_ASSIGN(auto input, file_system_->OpenInputFile("part=a/part0.parquet")); |
| 187 | + |
| 188 | + // Try to read metadata without providing decryption properties |
| 189 | + // when the footer is encrypted. |
| 190 | + ASSERT_THROW(parquet::ReadMetaData(input), parquet::ParquetException); |
| 191 | + |
| 192 | + // Create the ReaderProperties object using the FileDecryptionProperties object |
| 193 | + auto decryption_config = |
| 194 | + std::make_shared<parquet::encryption::DecryptionConfiguration>(); |
| 195 | + auto file_decryption_properties = crypto_factory_->GetFileDecryptionProperties( |
| 196 | + *kms_connection_config_, *decryption_config); |
| 197 | + auto reader_properties = parquet::default_reader_properties(); |
| 198 | + reader_properties.file_decryption_properties(file_decryption_properties); |
| 199 | + |
| 200 | + // Read entire file as a single Arrow table |
| 201 | + parquet::arrow::FileReaderBuilder reader_builder; |
| 202 | + ASSERT_OK(reader_builder.Open(input, reader_properties)); |
| 203 | + ASSERT_OK_AND_ASSIGN(auto arrow_reader, reader_builder.Build()); |
| 204 | + std::shared_ptr<Table> table; |
| 205 | + ASSERT_OK(arrow_reader->ReadTable(&table)); |
| 206 | + |
| 207 | + // Check the contents of the table |
| 208 | + ASSERT_EQ(table->num_rows(), 2); |
| 209 | + ASSERT_EQ(table->num_columns(), 3); |
| 210 | + ASSERT_EQ(checked_pointer_cast<Int64Array>(table->column(0)->chunk(0))->GetView(0), 0); |
| 211 | + ASSERT_EQ(checked_pointer_cast<Int64Array>(table->column(1)->chunk(0))->GetView(0), 9); |
| 212 | + ASSERT_EQ(checked_pointer_cast<Int64Array>(table->column(2)->chunk(0))->GetView(0), 1); |
| 213 | +} |
| 214 | + |
| 215 | +} // namespace dataset |
| 216 | +} // namespace arrow |
0 commit comments