1919
2020#include " gtest/gtest.h"
2121
22- #include < arrow/dataset/dataset.h>
23- #include < arrow/dataset/file_base.h>
24- #include < arrow/dataset/file_parquet.h>
2522#include " arrow/array.h"
23+ #include " arrow/dataset/dataset.h"
24+ #include " arrow/dataset/file_base.h"
25+ #include " arrow/dataset/file_parquet.h"
2626#include " arrow/dataset/parquet_encryption_config.h"
2727#include " arrow/dataset/partition.h"
2828#include " arrow/filesystem/mockfs.h"
2929#include " arrow/io/api.h"
3030#include " arrow/status.h"
3131#include " arrow/table.h"
3232#include " arrow/testing/gtest_util.h"
33+ #include " arrow/testing/random.h"
3334#include " arrow/type.h"
3435#include " parquet/arrow/reader.h"
3536#include " parquet/encryption/crypto_factory.h"
36- #include " parquet/encryption/encryption.h"
3737#include " parquet/encryption/encryption_internal.h"
3838#include " parquet/encryption/kms_client.h"
3939#include " parquet/encryption/test_in_memory_kms.h"
@@ -51,14 +51,13 @@ using arrow::internal::checked_pointer_cast;
5151namespace arrow {
5252namespace dataset {
5353
54- class DatasetEncryptionTest : public ::testing::Test {
54+ class DatasetEncryptionTestBase : public ::testing::Test {
5555 public:
5656 // This function creates a mock file system using the current time point, creates a
5757 // directory with the given base directory path, and writes a dataset to it using
58- // provided Parquet file write options. The dataset is partitioned using a Hive
59- // partitioning scheme. The function also checks if the written files exist in the file
60- // system.
61- static void SetUpTestSuite () {
58+ // provided Parquet file write options. The function also checks if the written files
59+ // exist in the file system.
60+ void SetUp () override {
6261#ifdef ARROW_VALGRIND
6362 // Not necessary otherwise, but prevents a Valgrind leak by making sure
6463 // OpenSSL initialization is done from the main thread
@@ -71,24 +70,8 @@ class DatasetEncryptionTest : public ::testing::Test {
7170 std::chrono::system_clock::now (), {}));
7271 ASSERT_OK (file_system_->CreateDir (std::string (kBaseDir )));
7372
74- // Prepare table data.
75- auto table_schema = schema ({field (" a" , int64 ()), field (" c" , int64 ()),
76- field (" e" , int64 ()), field (" part" , utf8 ())});
77- table_ = TableFromJSON (table_schema, {R"( [
78- [ 0, 9, 1, "a" ],
79- [ 1, 8, 2, "a" ],
80- [ 2, 7, 1, "c" ],
81- [ 3, 6, 2, "c" ],
82- [ 4, 5, 1, "e" ],
83- [ 5, 4, 2, "e" ],
84- [ 6, 3, 1, "g" ],
85- [ 7, 2, 2, "g" ],
86- [ 8, 1, 1, "i" ],
87- [ 9, 0, 2, "i" ]
88- ])" });
89-
90- // Use a Hive-style partitioning scheme.
91- partitioning_ = std::make_shared<HivePartitioning>(schema ({field (" part" , utf8 ())}));
73+ // Init dataset and partitioning.
74+ ASSERT_NO_FATAL_FAILURE (PrepareTableAndPartitioning ());
9275
9376 // Prepare encryption properties.
9477 std::unordered_map<std::string, std::string> key_map;
@@ -133,13 +116,81 @@ class DatasetEncryptionTest : public ::testing::Test {
133116 ASSERT_OK (FileSystemDataset::Write (write_options, std::move (scanner)));
134117 }
135118
119+ virtual void PrepareTableAndPartitioning () = 0;
120+
121+ void TestScanDataset () {
122+ // Create decryption properties.
123+ auto decryption_config =
124+ std::make_shared<parquet::encryption::DecryptionConfiguration>();
125+ auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
126+ parquet_decryption_config->crypto_factory = crypto_factory_;
127+ parquet_decryption_config->kms_connection_config = kms_connection_config_;
128+ parquet_decryption_config->decryption_config = std::move (decryption_config);
129+
130+ // Set scan options.
131+ auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
132+ parquet_scan_options->parquet_decryption_config =
133+ std::move (parquet_decryption_config);
134+
135+ auto file_format = std::make_shared<ParquetFileFormat>();
136+ file_format->default_fragment_scan_options = std::move (parquet_scan_options);
137+
138+ // Get FileInfo objects for all files under the base directory
139+ fs::FileSelector selector;
140+ selector.base_dir = kBaseDir ;
141+ selector.recursive = true ;
142+
143+ FileSystemFactoryOptions factory_options;
144+ factory_options.partitioning = partitioning_;
145+ factory_options.partition_base_dir = kBaseDir ;
146+ ASSERT_OK_AND_ASSIGN (auto dataset_factory,
147+ FileSystemDatasetFactory::Make (file_system_, selector,
148+ file_format, factory_options));
149+
150+ // Read dataset into table
151+ ASSERT_OK_AND_ASSIGN (auto dataset, dataset_factory->Finish ());
152+ ASSERT_OK_AND_ASSIGN (auto scanner_builder, dataset->NewScan ());
153+ ASSERT_OK_AND_ASSIGN (auto scanner, scanner_builder->Finish ());
154+ ASSERT_OK_AND_ASSIGN (auto read_table, scanner->ToTable ());
155+
156+ // Verify the data was read correctly
157+ ASSERT_OK_AND_ASSIGN (auto combined_table, read_table->CombineChunks ());
158+ // Validate the table
159+ ASSERT_OK (combined_table->ValidateFull ());
160+ AssertTablesEqual (*combined_table, *table_);
161+ }
162+
136163 protected:
137- inline static std::shared_ptr<fs::FileSystem> file_system_;
138- inline static std::shared_ptr<Table> table_;
139- inline static std::shared_ptr<HivePartitioning> partitioning_;
140- inline static std::shared_ptr<parquet::encryption::CryptoFactory> crypto_factory_;
141- inline static std::shared_ptr<parquet::encryption::KmsConnectionConfig>
142- kms_connection_config_;
164+ std::shared_ptr<fs::FileSystem> file_system_;
165+ std::shared_ptr<Table> table_;
166+ std::shared_ptr<Partitioning> partitioning_;
167+ std::shared_ptr<parquet::encryption::CryptoFactory> crypto_factory_;
168+ std::shared_ptr<parquet::encryption::KmsConnectionConfig> kms_connection_config_;
169+ };
170+
171+ class DatasetEncryptionTest : public DatasetEncryptionTestBase {
172+ public:
173+ // The dataset is partitioned using a Hive partitioning scheme.
174+ void PrepareTableAndPartitioning () override {
175+ // Prepare table data.
176+ auto table_schema = schema ({field (" a" , int64 ()), field (" c" , int64 ()),
177+ field (" e" , int64 ()), field (" part" , utf8 ())});
178+ table_ = TableFromJSON (table_schema, {R"( [
179+ [ 0, 9, 1, "a" ],
180+ [ 1, 8, 2, "a" ],
181+ [ 2, 7, 1, "c" ],
182+ [ 3, 6, 2, "c" ],
183+ [ 4, 5, 1, "e" ],
184+ [ 5, 4, 2, "e" ],
185+ [ 6, 3, 1, "g" ],
186+ [ 7, 2, 2, "g" ],
187+ [ 8, 1, 1, "i" ],
188+ [ 9, 0, 2, "i" ]
189+ ])" });
190+
191+ // Use a Hive-style partitioning scheme.
192+ partitioning_ = std::make_shared<HivePartitioning>(schema ({field (" part" , utf8 ())}));
193+ }
143194};
144195
145196// This test demonstrates the process of writing a partitioned Parquet file with the same
@@ -148,44 +199,7 @@ class DatasetEncryptionTest : public ::testing::Test {
148199// test reads the data back and verifies that it can be successfully decrypted and
149200// scanned.
150201TEST_F (DatasetEncryptionTest, WriteReadDatasetWithEncryption) {
151- // Create decryption properties.
152- auto decryption_config =
153- std::make_shared<parquet::encryption::DecryptionConfiguration>();
154- auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
155- parquet_decryption_config->crypto_factory = crypto_factory_;
156- parquet_decryption_config->kms_connection_config = kms_connection_config_;
157- parquet_decryption_config->decryption_config = std::move (decryption_config);
158-
159- // Set scan options.
160- auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
161- parquet_scan_options->parquet_decryption_config = std::move (parquet_decryption_config);
162-
163- auto file_format = std::make_shared<ParquetFileFormat>();
164- file_format->default_fragment_scan_options = std::move (parquet_scan_options);
165-
166- // Get FileInfo objects for all files under the base directory
167- fs::FileSelector selector;
168- selector.base_dir = kBaseDir ;
169- selector.recursive = true ;
170-
171- FileSystemFactoryOptions factory_options;
172- factory_options.partitioning = partitioning_;
173- factory_options.partition_base_dir = kBaseDir ;
174- ASSERT_OK_AND_ASSIGN (auto dataset_factory,
175- FileSystemDatasetFactory::Make (file_system_, selector, file_format,
176- factory_options));
177-
178- // Read dataset into table
179- ASSERT_OK_AND_ASSIGN (auto dataset, dataset_factory->Finish ());
180- ASSERT_OK_AND_ASSIGN (auto scanner_builder, dataset->NewScan ());
181- ASSERT_OK_AND_ASSIGN (auto scanner, scanner_builder->Finish ());
182- ASSERT_OK_AND_ASSIGN (auto read_table, scanner->ToTable ());
183-
184- // Verify the data was read correctly
185- ASSERT_OK_AND_ASSIGN (auto combined_table, read_table->CombineChunks ());
186- // Validate the table
187- ASSERT_OK (combined_table->ValidateFull ());
188- AssertTablesEqual (*combined_table, *table_);
202+ ASSERT_NO_FATAL_FAILURE (TestScanDataset ());
189203}
190204
191205// Read a single parquet file with and without decryption properties.
@@ -220,5 +234,26 @@ TEST_F(DatasetEncryptionTest, ReadSingleFile) {
220234 ASSERT_EQ (checked_pointer_cast<Int64Array>(table->column (2 )->chunk (0 ))->GetView (0 ), 1 );
221235}
222236
237+ class LargeRowEncryptionTest : public DatasetEncryptionTestBase {
238+ public:
239+ // The dataset is partitioned using a Hive partitioning scheme.
240+ void PrepareTableAndPartitioning () override {
241+ // Create a random floating-point array with large number of rows.
242+ constexpr int kRowCount = 32769 ;
243+ arrow::random::RandomArrayGenerator rand_gen (0 );
244+ auto array = rand_gen.Float32 (kRowCount , 0.0 , 1.0 , false );
245+ auto table_schema = schema ({field (" a" , float32 ())});
246+
247+ // Prepare table and partitioning.
248+ table_ = arrow::Table::Make (table_schema, {array});
249+ partitioning_ = std::make_shared<dataset::DirectoryPartitioning>(arrow::schema ({}));
250+ }
251+ };
252+
253+ // Test for writing and reading encrypted dataset with large row count.
254+ TEST_F (LargeRowEncryptionTest, ReadEncryptLargeRows) {
255+ ASSERT_NO_FATAL_FAILURE (TestScanDataset ());
256+ }
257+
223258} // namespace dataset
224259} // namespace arrow
0 commit comments