|
19 | 19 |
|
20 | 20 | #include <stdio.h> |
21 | 21 |
|
| 22 | +#include <stdexcept> |
| 23 | +#include <tuple> |
| 24 | +#include <vector> |
| 25 | + |
22 | 26 | #include <arrow/io/file.h> |
23 | 27 |
|
| 28 | +#include "arrow/util/endian.h" |
| 29 | +#include "arrow/util/ubsan.h" |
| 30 | + |
24 | 31 | #include "parquet/column_reader.h" |
25 | 32 | #include "parquet/column_writer.h" |
| 33 | +#include "parquet/encryption/encryption_internal.h" |
26 | 34 | #include "parquet/encryption/test_encryption_util.h" |
27 | 35 | #include "parquet/file_reader.h" |
28 | 36 | #include "parquet/file_writer.h" |
@@ -195,6 +203,170 @@ TEST_F(TestEncryptionConfiguration, EncryptTwoColumnsAndFooterUseAES_GCM_CTR) { |
195 | 203 | "tmp_encrypt_columns_and_footer_ctr.parquet.encrypted")); |
196 | 204 | } |
197 | 205 |
|
| 206 | +enum class AadPrefixMode { None, Stored, External }; |
| 207 | + |
| 208 | +class TestGeneratedEncryptionConfiguration |
| 209 | + : public ::testing::TestWithParam< |
| 210 | + std::tuple<ParquetCipher::type, bool, AadPrefixMode>> { |
| 211 | + public: |
| 212 | + void SetUp() override { |
| 213 | + PARQUET_ASSIGN_OR_THROW(temp_dir_, temp_data_dir()); |
| 214 | + file_path_ = temp_dir_->path().ToString() + "algorithm_metadata.parquet.encrypted"; |
| 215 | + |
| 216 | + std::map<std::string, std::shared_ptr<ColumnEncryptionProperties>> columns; |
| 217 | + columns[kDoubleFieldName] = |
| 218 | + ColumnEncryptionProperties::WithColumnKey(kColumnEncryptionKey1, "kc1"); |
| 219 | + columns[kFloatFieldName] = |
| 220 | + ColumnEncryptionProperties::WithColumnKey(kColumnEncryptionKey2, "kc2"); |
| 221 | + columns[kBooleanFieldName] = ColumnEncryptionProperties::Unencrypted(); |
| 222 | + |
| 223 | + FileEncryptionProperties::Builder builder(kFooterEncryptionKey); |
| 224 | + builder.footer_key_metadata("kf") |
| 225 | + ->encrypted_columns(std::move(columns)) |
| 226 | + ->algorithm(std::get<0>(GetParam())); |
| 227 | + if (std::get<1>(GetParam())) { |
| 228 | + builder.set_plaintext_footer(); |
| 229 | + } |
| 230 | + if (std::get<2>(GetParam()) != AadPrefixMode::None) { |
| 231 | + builder.aad_prefix(kFileName); |
| 232 | + } |
| 233 | + if (std::get<2>(GetParam()) == AadPrefixMode::External) { |
| 234 | + builder.disable_aad_prefix_storage(); |
| 235 | + } |
| 236 | + auto properties = builder.build(); |
| 237 | + file_aad_ = properties->file_aad(); |
| 238 | + FileEncryptor().EncryptFile(file_path_, std::move(properties)); |
| 239 | + } |
| 240 | + |
| 241 | + protected: |
| 242 | + enum class Keys { All, MissingColumn, WrongFooter }; |
| 243 | + enum class AadPrefix { Match, Missing, Wrong }; |
| 244 | + |
| 245 | + std::shared_ptr<FileDecryptionProperties> MakeDecryptionProperties( |
| 246 | + Keys keys = Keys::All, AadPrefix prefix = AadPrefix::Match) { |
| 247 | + auto retriever = std::make_shared<StringKeyIdRetriever>(); |
| 248 | + retriever->PutKey("kf", keys == Keys::WrongFooter ? SecureString(kNewFooterMasterKey) |
| 249 | + : kFooterEncryptionKey); |
| 250 | + if (keys != Keys::MissingColumn) { |
| 251 | + retriever->PutKey("kc1", kColumnEncryptionKey1); |
| 252 | + } |
| 253 | + retriever->PutKey("kc2", kColumnEncryptionKey2); |
| 254 | + |
| 255 | + FileDecryptionProperties::Builder builder; |
| 256 | + builder.key_retriever(std::move(retriever)); |
| 257 | + if (std::get<2>(GetParam()) != AadPrefixMode::None && prefix != AadPrefix::Missing) { |
| 258 | + builder.aad_prefix(prefix == AadPrefix::Wrong ? "incorrect-prefix" : kFileName); |
| 259 | + } |
| 260 | + return builder.build(); |
| 261 | + } |
| 262 | + |
| 263 | + std::unique_ptr<ParquetFileReader> OpenReader( |
| 264 | + std::shared_ptr<FileDecryptionProperties> decryption_properties) { |
| 265 | + auto reader_properties = default_reader_properties(); |
| 266 | + reader_properties.file_decryption_properties(std::move(decryption_properties)); |
| 267 | + return ParquetFileReader::OpenFile(file_path_, false, reader_properties); |
| 268 | + } |
| 269 | + |
| 270 | + void CheckPlaintextFooterSignature() { |
| 271 | + PARQUET_ASSIGN_OR_THROW(auto file, ::arrow::io::ReadableFile::Open(file_path_)); |
| 272 | + PARQUET_ASSIGN_OR_THROW(auto file_size, file->GetSize()); |
| 273 | + ASSERT_GT(file_size, 8); |
| 274 | + PARQUET_ASSIGN_OR_THROW(auto trailer, file->ReadAt(file_size - 8, 8)); |
| 275 | + ASSERT_EQ(trailer->size(), 8); |
| 276 | + const auto footer_size = ::arrow::bit_util::FromLittleEndian( |
| 277 | + ::arrow::util::SafeLoadAs<uint32_t>(trailer->data())); |
| 278 | + constexpr auto signature_size = kNonceLength + kGcmTagLength; |
| 279 | + ASSERT_GT(footer_size, signature_size); |
| 280 | + ASSERT_LE(footer_size, file_size - 8); |
| 281 | + PARQUET_ASSIGN_OR_THROW(auto footer, |
| 282 | + file->ReadAt(file_size - 8 - footer_size, footer_size)); |
| 283 | + ASSERT_EQ(footer->size(), footer_size); |
| 284 | + const auto footer_span = footer->span_as<uint8_t>(); |
| 285 | + const auto metadata = footer_span.first(footer_size - signature_size); |
| 286 | + const auto nonce = footer_span.subspan(metadata.size(), kNonceLength); |
| 287 | + const auto tag = footer_span.last(kGcmTagLength); |
| 288 | + // Footer AAD is the file AAD followed by the footer module type byte. |
| 289 | + const auto aad = file_aad_ + static_cast<char>(kFooter); |
| 290 | + |
| 291 | + // Verify the on-disk tag with GCM explicitly, independently of reader selection. |
| 292 | + auto encryptor = AesEncryptor::Make(ParquetCipher::AES_GCM_V1, |
| 293 | + static_cast<int32_t>(kFooterEncryptionKey.size()), |
| 294 | + /*metadata=*/true, |
| 295 | + /*write_length=*/false); |
| 296 | + std::vector<uint8_t> encrypted(encryptor->CiphertextLength(metadata.size())); |
| 297 | + encryptor->SignedFooterEncrypt(metadata, kFooterEncryptionKey.as_span(), |
| 298 | + str2span(aad), nonce, encrypted); |
| 299 | + EXPECT_EQ(std::vector<uint8_t>(tag.begin(), tag.end()), |
| 300 | + std::vector<uint8_t>(encrypted.end() - kGcmTagLength, encrypted.end())); |
| 301 | + } |
| 302 | + |
| 303 | + void ReadFirstSecretValue(std::shared_ptr<FileDecryptionProperties> properties) { |
| 304 | + auto reader = OpenReader(std::move(properties)); |
| 305 | + auto column = reader->RowGroup(0)->Column(5); |
| 306 | + auto* double_reader = static_cast<DoubleReader*>(column.get()); |
| 307 | + double value; |
| 308 | + int64_t values_read; |
| 309 | + double_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read); |
| 310 | + } |
| 311 | + |
| 312 | + std::unique_ptr<TemporaryDir> temp_dir_; |
| 313 | + std::string file_path_; |
| 314 | + std::string file_aad_; |
| 315 | +}; |
| 316 | + |
| 317 | +TEST_P(TestGeneratedEncryptionConfiguration, AlgorithmMetadataAndRoundTrip) { |
| 318 | + const auto cipher = std::get<0>(GetParam()); |
| 319 | + const bool plaintext_footer = std::get<1>(GetParam()); |
| 320 | + const auto prefix_mode = std::get<2>(GetParam()); |
| 321 | + if (plaintext_footer) { |
| 322 | + CheckPlaintextFooterSignature(); |
| 323 | + } |
| 324 | + auto reader = OpenReader(MakeDecryptionProperties()); |
| 325 | + const auto metadata = reader->metadata(); |
| 326 | + if (plaintext_footer) { |
| 327 | + ASSERT_TRUE(metadata->is_encryption_algorithm_set()); |
| 328 | + const auto algorithm = metadata->encryption_algorithm(); |
| 329 | + ASSERT_EQ(cipher, algorithm.algorithm); |
| 330 | + EXPECT_FALSE(algorithm.aad.aad_file_unique.empty()); |
| 331 | + EXPECT_EQ(prefix_mode == AadPrefixMode::External, algorithm.aad.supply_aad_prefix); |
| 332 | + EXPECT_EQ(prefix_mode == AadPrefixMode::Stored ? kFileName : "", |
| 333 | + algorithm.aad.aad_prefix); |
| 334 | + } else { |
| 335 | + // An encrypted footer declares the algorithm in FileCryptoMetaData instead. |
| 336 | + EXPECT_FALSE(metadata->is_encryption_algorithm_set()); |
| 337 | + } |
| 338 | + reader->Close(); |
| 339 | + |
| 340 | + FileDecryptor decryptor; |
| 341 | + // Verify every physical column and all row groups through sync and async readers. |
| 342 | + ASSERT_NO_THROW(decryptor.DecryptFile(file_path_, MakeDecryptionProperties())); |
| 343 | + if (prefix_mode == AadPrefixMode::Stored) { |
| 344 | + ASSERT_NO_THROW(decryptor.DecryptFile( |
| 345 | + file_path_, MakeDecryptionProperties(Keys::All, AadPrefix::Missing))); |
| 346 | + } |
| 347 | + if (plaintext_footer) { |
| 348 | + // This reads only public columns and does not authenticate the keyless footer. |
| 349 | + ASSERT_NO_THROW(decryptor.DecryptFile(file_path_, nullptr)); |
| 350 | + } |
| 351 | + EXPECT_THROW(ReadFirstSecretValue(MakeDecryptionProperties(Keys::MissingColumn)), |
| 352 | + std::out_of_range); |
| 353 | + EXPECT_THROW(OpenReader(MakeDecryptionProperties(Keys::WrongFooter)), ParquetException); |
| 354 | + if (prefix_mode == AadPrefixMode::External) { |
| 355 | + EXPECT_THROW(OpenReader(MakeDecryptionProperties(Keys::All, AadPrefix::Missing)), |
| 356 | + ParquetException); |
| 357 | + EXPECT_THROW(OpenReader(MakeDecryptionProperties(Keys::All, AadPrefix::Wrong)), |
| 358 | + ParquetException); |
| 359 | + } |
| 360 | +} |
| 361 | + |
| 362 | +INSTANTIATE_TEST_SUITE_P( |
| 363 | + GeneratedFiles, TestGeneratedEncryptionConfiguration, |
| 364 | + ::testing::Combine(::testing::Values(ParquetCipher::AES_GCM_V1, |
| 365 | + ParquetCipher::AES_GCM_CTR_V1), |
| 366 | + ::testing::Bool(), |
| 367 | + ::testing::Values(AadPrefixMode::None, AadPrefixMode::Stored, |
| 368 | + AadPrefixMode::External))); |
| 369 | + |
198 | 370 | // Set temp_dir before running the write/read tests. The encrypted files will |
199 | 371 | // be written/read from this directory. |
200 | 372 | void TestEncryptionConfiguration::SetUpTestCase() { |
|
0 commit comments