Skip to content

Commit a40c629

Browse files
author
Laurens Kuiper
authored
Enable Parquet reader support for uniformly encrypted Parquet files written by Arrow C++ API (#20046)
This PR enables to read _uniformly encrypted parquet files_ written by the Arrow C++ API as addressed in apache/arrow#47254. Fixes #18524 and https://github.com/duckdblabs/duckdb-internal/issues/5511. Previously, we were not able to read uniformly encrypted parquet files written with Arrow since we did not use the additional authenticated data (aad). The aad consist of a unique prefix and a suffix. The suffix consists of the module type, row group ordinal (present in metadata), column ordinal (column index of the _original_ scheme of a parquet file) and a page ordinal (page index). More details can be found in the [Parquet Encryption Spec](https://parquet.apache.org/docs/file-format/data-pages/encryption/). For testing I've added the uniformly encrypted file from the [parquet-testing](https://github.com/apache/parquet-testing/tree/master/data) repo and used the [gist](https://gist.github.com/rok/8a68066c51801458a3746772a2c736c5) of @rok. On top of this, I've added an Arrow file containing multiple row groups based on the previous gist ([code](https://gist.github.com/ccfelius/3def798ade5de006e6d0073c1c457990)). As also mentioned in apache/arrow#47254, PyArrow uses a KMS which creates randomly generated keys. This means that DuckDB is not able to read files written with PyArrow, since keys are used directly in DuckDB. The fix ensures backwards compatibility with our own encrypted parquet files. In a follow up PR I'll address the parquet writer.
2 parents a511415 + 9ef7203 commit a40c629

14 files changed

Lines changed: 521 additions & 42 deletions
5.57 KB
Binary file not shown.
Binary file not shown.
Binary file not shown.

‎extension/parquet/column_reader.cpp‎

Lines changed: 62 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
#include "duckdb/common/helper.hpp"
2929
#include "duckdb/common/types/bit.hpp"
3030

31+
#include "parquet_crypto.hpp"
32+
3133
namespace duckdb {
3234

3335
using duckdb_parquet::CompressionCodec;
@@ -109,7 +111,7 @@ const uint8_t ParquetDecodeUtils::BITPACK_DLEN = 8;
109111
ColumnReader::ColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema_p)
110112
: column_schema(schema_p), reader(reader), page_rows_available(0), dictionary_decoder(*this),
111113
delta_binary_packed_decoder(*this), rle_decoder(*this), delta_length_byte_array_decoder(*this),
112-
delta_byte_array_decoder(*this), byte_stream_split_decoder(*this) {
114+
delta_byte_array_decoder(*this), byte_stream_split_decoder(*this), aad_crypto_metadata(reader.allocator) {
113115
}
114116

115117
ColumnReader::~ColumnReader() {
@@ -232,23 +234,54 @@ bool ColumnReader::PageIsFilteredOut(PageHeader &page_hdr) {
232234
return true;
233235
}
234236

237+
void ColumnReader::ReadEncrypted(duckdb_apache::thrift::TBase &object) {
238+
aad_crypto_metadata.module = ParquetCrypto::GetModuleHeader(*chunk, aad_crypto_metadata.page_ordinal);
239+
aad_crypto_metadata.page_ordinal =
240+
ParquetCrypto::GetFinalPageOrdinal(*chunk, aad_crypto_metadata.module, aad_crypto_metadata.page_ordinal);
241+
reader.ReadEncrypted(object, *protocol, aad_crypto_metadata);
242+
}
243+
244+
void ColumnReader::ReadDataEncrypted(const data_ptr_t buffer, const uint32_t buffer_size, PageType::type page_type) {
245+
aad_crypto_metadata.module = ParquetCrypto::GetModule(*chunk, page_type, aad_crypto_metadata.page_ordinal);
246+
aad_crypto_metadata.page_ordinal =
247+
ParquetCrypto::GetFinalPageOrdinal(*chunk, aad_crypto_metadata.module, aad_crypto_metadata.page_ordinal);
248+
reader.ReadDataEncrypted(*protocol, buffer, buffer_size, aad_crypto_metadata);
249+
}
250+
251+
void ColumnReader::Read(PageHeader &page_hdr) {
252+
if (reader.parquet_options.encryption_config) {
253+
ReadEncrypted(page_hdr);
254+
} else {
255+
reader.Read(page_hdr, *protocol);
256+
}
257+
}
258+
259+
void ColumnReader::ReadData(const data_ptr_t buffer, const uint32_t buffer_size, PageType::type page_type) {
260+
if (reader.parquet_options.encryption_config) {
261+
ReadDataEncrypted(buffer, buffer_size, page_type);
262+
} else {
263+
reader.ReadData(*protocol, buffer, buffer_size);
264+
}
265+
}
266+
235267
void ColumnReader::PrepareRead(optional_ptr<const TableFilter> filter, optional_ptr<TableFilterState> filter_state) {
236268
encoding = ColumnEncoding::INVALID;
237269
defined_decoder.reset();
238270
page_is_filtered_out = false;
239271
block.reset();
240272
PageHeader page_hdr;
241273
auto &trans = reinterpret_cast<ThriftFileTransport &>(*protocol->getTransport());
274+
242275
if (trans.HasPrefetch()) {
243276
// Already has some data prefetched, let's not mess with it
244-
reader.Read(page_hdr, *protocol);
277+
Read(page_hdr);
245278
} else {
246279
// No prefetch yet, prefetch the full header in one go (so thrift won't read byte-by-byte from storage)
247280
// 256 bytes should cover almost all headers (unless it's a V2 header with really LONG string statistics)
248281
static constexpr idx_t ASSUMED_HEADER_SIZE = 256;
249282
const auto prefetch_size = MinValue(trans.GetSize() - trans.GetLocation(), ASSUMED_HEADER_SIZE);
250283
trans.Prefetch(trans.GetLocation(), prefetch_size);
251-
reader.Read(page_hdr, *protocol);
284+
Read(page_hdr);
252285
trans.ClearPrefetch();
253286
}
254287
// some basic sanity check
@@ -304,7 +337,7 @@ void ColumnReader::PreparePageV2(PageHeader &page_hdr) {
304337
uncompressed = true;
305338
}
306339
if (uncompressed) {
307-
reader.ReadData(*protocol, block->ptr, page_hdr.compressed_page_size);
340+
ReadData(block->ptr, page_hdr.compressed_page_size, page_hdr.type);
308341
return;
309342
}
310343

@@ -317,14 +350,16 @@ void ColumnReader::PreparePageV2(PageHeader &page_hdr) {
317350
"repetition_levels_byte_length + definition_levels_byte_length",
318351
Reader().GetFileName());
319352
}
320-
reader.ReadData(*protocol, block->ptr, uncompressed_bytes);
353+
354+
ReadData(block->ptr, uncompressed_bytes, page_hdr.type);
321355

322356
auto compressed_bytes = page_hdr.compressed_page_size - uncompressed_bytes;
323357

324358
if (compressed_bytes > 0) {
325359
ResizeableBuffer compressed_buffer;
326360
compressed_buffer.resize(GetAllocator(), compressed_bytes);
327-
reader.ReadData(*protocol, compressed_buffer.ptr, compressed_bytes);
361+
362+
ReadData(compressed_buffer.ptr, compressed_bytes, page_hdr.type);
328363

329364
DecompressInternal(chunk->meta_data.codec, compressed_buffer.ptr, compressed_bytes,
330365
block->ptr + uncompressed_bytes, page_hdr.uncompressed_page_size - uncompressed_bytes);
@@ -341,19 +376,32 @@ void ColumnReader::AllocateBlock(idx_t size) {
341376

342377
void ColumnReader::PreparePage(PageHeader &page_hdr) {
343378
AllocateBlock(page_hdr.uncompressed_page_size + 1);
379+
uint32_t compressed_page_size = page_hdr.compressed_page_size;
380+
381+
if (chunk->__isset.crypto_metadata) {
382+
auto const file_aad = reader.GetUniqueFileIdentifier(reader.metadata->crypto_metadata->encryption_algorithm);
383+
if (!file_aad.empty()) {
384+
// If there is a file aad (identifier), this means that the Encrypted file is written by Arrow
385+
// Arrow adds the bytes for encryption (len + nonce + tag)
386+
// to the compressed page size
387+
compressed_page_size -=
388+
(ParquetCrypto::LENGTH_BYTES + ParquetCrypto::NONCE_BYTES + ParquetCrypto::TAG_BYTES);
389+
}
390+
}
391+
344392
if (chunk->meta_data.codec == CompressionCodec::UNCOMPRESSED) {
345-
if (page_hdr.compressed_page_size != page_hdr.uncompressed_page_size) {
393+
if (compressed_page_size != page_hdr.uncompressed_page_size) {
346394
throw std::runtime_error("Page size mismatch");
347395
}
348-
reader.ReadData(*protocol, block->ptr, page_hdr.compressed_page_size);
396+
ReadData(block->ptr, compressed_page_size, page_hdr.type);
349397
return;
350398
}
351399

352400
ResizeableBuffer compressed_buffer;
353-
compressed_buffer.resize(GetAllocator(), page_hdr.compressed_page_size + 1);
354-
reader.ReadData(*protocol, compressed_buffer.ptr, page_hdr.compressed_page_size);
401+
compressed_buffer.resize(GetAllocator(), compressed_page_size + 1);
402+
ReadData(compressed_buffer.ptr, compressed_page_size, page_hdr.type);
355403

356-
DecompressInternal(chunk->meta_data.codec, compressed_buffer.ptr, page_hdr.compressed_page_size, block->ptr,
404+
DecompressInternal(chunk->meta_data.codec, compressed_buffer.ptr, compressed_page_size, block->ptr,
357405
page_hdr.uncompressed_page_size);
358406
}
359407

@@ -523,8 +571,11 @@ void ColumnReader::BeginRead(data_ptr_t define_out, data_ptr_t repeat_out) {
523571

524572
idx_t ColumnReader::ReadPageHeaders(idx_t max_read, optional_ptr<const TableFilter> filter,
525573
optional_ptr<TableFilterState> filter_state) {
574+
int8_t page_ordinal = 0;
526575
while (page_rows_available == 0) {
576+
aad_crypto_metadata.page_ordinal = page_ordinal;
527577
PrepareRead(filter, filter_state);
578+
page_ordinal++;
528579
}
529580
return MinValue<idx_t>(MinValue<idx_t>(max_read, page_rows_available), STANDARD_VECTOR_SIZE);
530581
}

‎extension/parquet/include/column_reader.hpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,13 @@
2121
#include "decoder/delta_length_byte_array_decoder.hpp"
2222
#include "decoder/delta_byte_array_decoder.hpp"
2323
#include "parquet_column_schema.hpp"
24+
#include "parquet_crypto.hpp"
2425

2526
#include "duckdb/common/operator/cast_operators.hpp"
2627
#include "duckdb/common/types/string_type.hpp"
2728
#include "duckdb/common/types/vector.hpp"
2829
#include "duckdb/common/types/vector_cache.hpp"
30+
#include "duckdb/common/encryption_functions.hpp"
2931

3032
namespace duckdb {
3133
class ParquetReader;
@@ -94,6 +96,20 @@ class ColumnReader {
9496
return column_schema.max_repeat;
9597
}
9698

99+
void InitializeCryptoMetadata(const duckdb_parquet::EncryptionAlgorithm &encryption_algorithm,
100+
idx_t row_group_ordinal_p) {
101+
std::string unique_file_identifier;
102+
if (encryption_algorithm.__isset.AES_GCM_V1) {
103+
unique_file_identifier = encryption_algorithm.AES_GCM_V1.aad_file_unique;
104+
} else if (encryption_algorithm.__isset.AES_GCM_CTR_V1) {
105+
throw InternalException("File is encrypted with AES_GCM_CTR_V1, but this is not supported by DuckDB");
106+
} else {
107+
throw InternalException("File is encrypted but no encryption algorithm is set");
108+
}
109+
110+
aad_crypto_metadata.Initialize(unique_file_identifier, row_group_ordinal_p, ColumnIndex());
111+
}
112+
97113
virtual idx_t FileOffset() const;
98114
virtual uint64_t TotalCompressedSize();
99115
virtual idx_t GroupRowsAvailable();
@@ -177,6 +193,10 @@ class ColumnReader {
177193
idx_t &approved_tuple_count);
178194
void DirectSelect(uint64_t num_values, data_ptr_t define_out, data_ptr_t repeat_out, Vector &result,
179195
const SelectionVector &sel, idx_t approved_tuple_count);
196+
void ReadEncrypted(duckdb_apache::thrift::TBase &object);
197+
void ReadDataEncrypted(const data_ptr_t buffer, const uint32_t buffer_size, PageType::type module);
198+
void Read(PageHeader &page_hdr);
199+
void ReadData(const data_ptr_t buffer, const uint32_t buffer_size, PageType::type page_type);
180200

181201
private:
182202
//! Check if a previous table filter has filtered out this page
@@ -315,6 +335,7 @@ class ColumnReader {
315335
DeltaLengthByteArrayDecoder delta_length_byte_array_decoder;
316336
DeltaByteArrayDecoder delta_byte_array_decoder;
317337
ByteStreamSplitDecoder byte_stream_split_decoder;
338+
CryptoMetaData aad_crypto_metadata;
318339

319340
//! Resizeable buffers used for the various encodings above
320341
ResizeableBuffer encoding_buffers[2];

‎extension/parquet/include/parquet_crypto.hpp‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,19 @@
99
#pragma once
1010

1111
#include "parquet_types.h"
12+
#include "duckdb/common/allocator.hpp"
1213
#include "duckdb/common/encryption_state.hpp"
14+
#include "duckdb/common/encryption_functions.hpp"
1315
#include "duckdb/storage/object_cache.hpp"
1416

1517
namespace duckdb {
18+
class ParquetAdditionalAuthenticatedData;
1619

1720
using duckdb_apache::thrift::TBase;
1821
using duckdb_apache::thrift::protocol::TProtocol;
19-
22+
using duckdb_parquet::ColumnChunk;
23+
using duckdb_parquet::PageType;
24+
class Allocator;
2025
class BufferedFileWriter;
2126

2227
class ParquetKeys : public ObjectCacheEntry {
@@ -36,6 +41,43 @@ class ParquetKeys : public ObjectCacheEntry {
3641
unordered_map<string, string> keys;
3742
};
3843

44+
struct CryptoMetaData {
45+
CryptoMetaData(Allocator &allocator);
46+
void Initialize(const std::string &unique_file_identifier_p, int16_t row_group_ordinal = -1,
47+
int16_t column_ordinal = -1, int8_t module = -1, int16_t page_ordinal = -1);
48+
void ClearAdditionalAuthenticatedData();
49+
void SetModule(int8_t module_p);
50+
bool IsEmpty() const;
51+
52+
public:
53+
string unique_file_identifier = "";
54+
int8_t module;
55+
int16_t row_group_ordinal;
56+
int16_t column_ordinal;
57+
int16_t page_ordinal;
58+
59+
public:
60+
unique_ptr<ParquetAdditionalAuthenticatedData> additional_authenticated_data;
61+
};
62+
63+
class ParquetAdditionalAuthenticatedData : public AdditionalAuthenticatedData {
64+
public:
65+
explicit ParquetAdditionalAuthenticatedData(Allocator &allocator);
66+
~ParquetAdditionalAuthenticatedData() override;
67+
68+
public:
69+
idx_t GetPrefixSize() const;
70+
void Rewind() const;
71+
void WriteParquetAAD(const CryptoMetaData &crypto_meta_data);
72+
73+
private:
74+
void WritePrefix(const std::string &prefix);
75+
void WriteSuffix(const CryptoMetaData &crypto_meta_data);
76+
77+
private:
78+
optional_idx additional_authenticated_data_prefix_size;
79+
};
80+
3981
class ParquetEncryptionConfig {
4082
public:
4183
explicit ParquetEncryptionConfig();
@@ -68,22 +110,48 @@ class ParquetCrypto {
68110
static constexpr idx_t CRYPTO_BLOCK_SIZE = 4096;
69111
static constexpr idx_t BLOCK_SIZE = 16;
70112

113+
// Module types for encryption
114+
static constexpr int8_t FOOTER = 0;
115+
static constexpr int8_t COLUMN_METADATA = 1;
116+
static constexpr int8_t DATA_PAGE = 2;
117+
static constexpr int8_t DICTIONARY_PAGE = 3;
118+
static constexpr int8_t DATA_PAGE_HEADER = 4;
119+
static constexpr int8_t DICTIONARY_PAGE_HEADER = 5;
120+
static constexpr int8_t COLUMN_INDEX = 6;
121+
static constexpr int8_t OFFSET_INDEX = 7;
122+
static constexpr int8_t BLOOM_FILTER_HEADER = 8;
123+
static constexpr int8_t BLOOM_FILTER_BITSET = 9;
124+
125+
// Standard AAD length for file
126+
static constexpr int32_t UNIQUE_FILE_ID_LEN = 8;
127+
// Maximum Parquet AAD suffix bytes
128+
static constexpr int32_t AAD_MAX_SUFFIX_BYTES = 7;
129+
71130
public:
72131
//! Decrypt and read a Thrift object from the transport protocol
73-
static uint32_t Read(TBase &object, TProtocol &iprot, const string &key, const EncryptionUtil &encryption_util_p);
132+
static uint32_t Read(TBase &object, TProtocol &iprot, const string &key, const EncryptionUtil &encryption_util_p,
133+
const CryptoMetaData &crypto_meta_data);
74134
//! Encrypt and write a Thrift object to the transport protocol
75135
static uint32_t Write(const TBase &object, TProtocol &oprot, const string &key,
76136
const EncryptionUtil &encryption_util_p);
77137
//! Decrypt and read a buffer
78138
static uint32_t ReadData(TProtocol &iprot, const data_ptr_t buffer, const uint32_t buffer_size, const string &key,
79-
const EncryptionUtil &encryption_util_p);
139+
const EncryptionUtil &encryption_util_p, const CryptoMetaData &crypto_meta_data);
80140
//! Encrypt and write a buffer to a file
81141
static uint32_t WriteData(TProtocol &oprot, const const_data_ptr_t buffer, const uint32_t buffer_size,
82142
const string &key, const EncryptionUtil &encryption_util_p);
83143

84144
public:
85145
static void AddKey(ClientContext &context, const FunctionParameters &parameters);
86146
static bool ValidKey(const std::string &key);
147+
148+
public:
149+
static int8_t GetModuleHeader(const ColumnChunk &chunk, uint16_t page_ordinal);
150+
static int8_t GetModule(const ColumnChunk &chunk, PageType::type page_type, uint16_t page_ordinal);
151+
static int16_t GetFinalPageOrdinal(const ColumnChunk &chunk, uint8_t module, uint16_t page_ordinal);
152+
static void GenerateAdditionalAuthenticatedData(Allocator &allocator, CryptoMetaData &aad_crypto_metadata);
153+
static unique_ptr<ParquetAdditionalAuthenticatedData> GenerateFooterAAD(Allocator &allocator,
154+
const std::string &unique_file_identifier);
87155
};
88156

89157
} // namespace duckdb

‎extension/parquet/include/parquet_file_metadata_cache.hpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@
1414

1515
namespace duckdb {
1616
struct CachingFileHandle;
17+
using duckdb_parquet::FileCryptoMetaData;
1718

1819
enum class ParquetCacheValidity { VALID, INVALID, UNKNOWN };
1920

2021
class ParquetFileMetadataCache : public ObjectCacheEntry {
2122
public:
2223
ParquetFileMetadataCache(unique_ptr<duckdb_parquet::FileMetaData> file_metadata, CachingFileHandle &handle,
23-
unique_ptr<GeoParquetFileMetadata> geo_metadata, idx_t footer_size);
24+
unique_ptr<GeoParquetFileMetadata> geo_metadata,
25+
unique_ptr<FileCryptoMetaData> crypto_metadata, idx_t footer_size);
2426
~ParquetFileMetadataCache() override = default;
2527

2628
//! Parquet file metadata
@@ -29,6 +31,9 @@ class ParquetFileMetadataCache : public ObjectCacheEntry {
2931
//! GeoParquet metadata
3032
unique_ptr<GeoParquetFileMetadata> geo_metadata;
3133

34+
//! Crypto metadata
35+
unique_ptr<FileCryptoMetaData> crypto_metadata;
36+
3237
//! Parquet footer size
3338
idx_t footer_size;
3439

‎extension/parquet/include/parquet_reader.hpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "duckdb.hpp"
1212
#include "duckdb/storage/caching_file_system.hpp"
1313
#include "duckdb/common/common.hpp"
14+
#include "duckdb/common/encryption_functions.hpp"
1415
#include "duckdb/common/encryption_state.hpp"
1516
#include "duckdb/common/exception.hpp"
1617
#include "duckdb/common/multi_file/base_file_reader.hpp"
@@ -178,10 +179,15 @@ class ParquetReader : public BaseFileReader {
178179
idx_t NumRowGroups() const;
179180

180181
const duckdb_parquet::FileMetaData *GetFileMetadata() const;
182+
string static GetUniqueFileIdentifier(const duckdb_parquet::EncryptionAlgorithm &encryption_algorithm);
181183

182184
uint32_t Read(duckdb_apache::thrift::TBase &object, TProtocol &iprot);
185+
uint32_t ReadEncrypted(duckdb_apache::thrift::TBase &object, TProtocol &iprot,
186+
CryptoMetaData &aad_crypto_metadata) const;
183187
uint32_t ReadData(duckdb_apache::thrift::protocol::TProtocol &iprot, const data_ptr_t buffer,
184188
const uint32_t buffer_size);
189+
uint32_t ReadDataEncrypted(duckdb_apache::thrift::protocol::TProtocol &iprot, const data_ptr_t buffer,
190+
const uint32_t buffer_size, CryptoMetaData &aad_crypto_metadata) const;
185191

186192
unique_ptr<BaseStatistics> ReadStatistics(const string &name);
187193

@@ -231,6 +237,8 @@ class ParquetReader : public BaseFileReader {
231237

232238
MultiFileColumnDefinition ParseColumnDefinition(const duckdb_parquet::FileMetaData &file_meta_data,
233239
ParquetColumnSchema &element);
240+
unique_ptr<AdditionalAuthenticatedData> GenerateAAD(uint8_t module_type, uint16_t row_group_ordinal,
241+
uint16_t column_ordinal, uint16_t page_ordinal) const;
234242

235243
private:
236244
unique_ptr<CachingFileHandle> file_handle;

0 commit comments

Comments
 (0)