Skip to content

Commit df0d607

Browse files
committed
[DO NOT MERGE] apacheGH-39444: Try to reproduce the test failure
1 parent 06935a3 commit df0d607

1 file changed

Lines changed: 106 additions & 71 deletions

File tree

‎cpp/src/arrow/dataset/file_parquet_encryption_test.cc‎

Lines changed: 106 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,21 @@
1919

2020
#include "gtest/gtest.h"
2121

22-
#include <arrow/dataset/dataset.h>
23-
#include <arrow/dataset/file_base.h>
24-
#include <arrow/dataset/file_parquet.h>
2522
#include "arrow/array.h"
23+
#include "arrow/dataset/dataset.h"
24+
#include "arrow/dataset/file_base.h"
25+
#include "arrow/dataset/file_parquet.h"
2626
#include "arrow/dataset/parquet_encryption_config.h"
2727
#include "arrow/dataset/partition.h"
2828
#include "arrow/filesystem/mockfs.h"
2929
#include "arrow/io/api.h"
3030
#include "arrow/status.h"
3131
#include "arrow/table.h"
3232
#include "arrow/testing/gtest_util.h"
33+
#include "arrow/testing/random.h"
3334
#include "arrow/type.h"
3435
#include "parquet/arrow/reader.h"
3536
#include "parquet/encryption/crypto_factory.h"
36-
#include "parquet/encryption/encryption.h"
3737
#include "parquet/encryption/encryption_internal.h"
3838
#include "parquet/encryption/kms_client.h"
3939
#include "parquet/encryption/test_in_memory_kms.h"
@@ -51,14 +51,13 @@ using arrow::internal::checked_pointer_cast;
5151
namespace arrow {
5252
namespace dataset {
5353

54-
class DatasetEncryptionTest : public ::testing::Test {
54+
class DatasetEncryptionTestBase : public ::testing::Test {
5555
public:
5656
// This function creates a mock file system using the current time point, creates a
5757
// directory with the given base directory path, and writes a dataset to it using
58-
// provided Parquet file write options. The dataset is partitioned using a Hive
59-
// partitioning scheme. The function also checks if the written files exist in the file
60-
// system.
61-
static void SetUpTestSuite() {
58+
// provided Parquet file write options. The function also checks if the written files
59+
// exist in the file system.
60+
void SetUp() override {
6261
#ifdef ARROW_VALGRIND
6362
// Not necessary otherwise, but prevents a Valgrind leak by making sure
6463
// OpenSSL initialization is done from the main thread
@@ -71,24 +70,8 @@ class DatasetEncryptionTest : public ::testing::Test {
7170
std::chrono::system_clock::now(), {}));
7271
ASSERT_OK(file_system_->CreateDir(std::string(kBaseDir)));
7372

74-
// Prepare table data.
75-
auto table_schema = schema({field("a", int64()), field("c", int64()),
76-
field("e", int64()), field("part", utf8())});
77-
table_ = TableFromJSON(table_schema, {R"([
78-
[ 0, 9, 1, "a" ],
79-
[ 1, 8, 2, "a" ],
80-
[ 2, 7, 1, "c" ],
81-
[ 3, 6, 2, "c" ],
82-
[ 4, 5, 1, "e" ],
83-
[ 5, 4, 2, "e" ],
84-
[ 6, 3, 1, "g" ],
85-
[ 7, 2, 2, "g" ],
86-
[ 8, 1, 1, "i" ],
87-
[ 9, 0, 2, "i" ]
88-
])"});
89-
90-
// Use a Hive-style partitioning scheme.
91-
partitioning_ = std::make_shared<HivePartitioning>(schema({field("part", utf8())}));
73+
// Init dataset and partitioning.
74+
ASSERT_NO_FATAL_FAILURE(PrepareTableAndPartitioning());
9275

9376
// Prepare encryption properties.
9477
std::unordered_map<std::string, std::string> key_map;
@@ -133,13 +116,81 @@ class DatasetEncryptionTest : public ::testing::Test {
133116
ASSERT_OK(FileSystemDataset::Write(write_options, std::move(scanner)));
134117
}
135118

119+
virtual void PrepareTableAndPartitioning() = 0;
120+
121+
void TestScanDataset() {
122+
// Create decryption properties.
123+
auto decryption_config =
124+
std::make_shared<parquet::encryption::DecryptionConfiguration>();
125+
auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
126+
parquet_decryption_config->crypto_factory = crypto_factory_;
127+
parquet_decryption_config->kms_connection_config = kms_connection_config_;
128+
parquet_decryption_config->decryption_config = std::move(decryption_config);
129+
130+
// Set scan options.
131+
auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
132+
parquet_scan_options->parquet_decryption_config =
133+
std::move(parquet_decryption_config);
134+
135+
auto file_format = std::make_shared<ParquetFileFormat>();
136+
file_format->default_fragment_scan_options = std::move(parquet_scan_options);
137+
138+
// Get FileInfo objects for all files under the base directory
139+
fs::FileSelector selector;
140+
selector.base_dir = kBaseDir;
141+
selector.recursive = true;
142+
143+
FileSystemFactoryOptions factory_options;
144+
factory_options.partitioning = partitioning_;
145+
factory_options.partition_base_dir = kBaseDir;
146+
ASSERT_OK_AND_ASSIGN(auto dataset_factory,
147+
FileSystemDatasetFactory::Make(file_system_, selector,
148+
file_format, factory_options));
149+
150+
// Read dataset into table
151+
ASSERT_OK_AND_ASSIGN(auto dataset, dataset_factory->Finish());
152+
ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
153+
ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
154+
ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable());
155+
156+
// Verify the data was read correctly
157+
ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks());
158+
// Validate the table
159+
ASSERT_OK(combined_table->ValidateFull());
160+
AssertTablesEqual(*combined_table, *table_);
161+
}
162+
136163
protected:
137-
inline static std::shared_ptr<fs::FileSystem> file_system_;
138-
inline static std::shared_ptr<Table> table_;
139-
inline static std::shared_ptr<HivePartitioning> partitioning_;
140-
inline static std::shared_ptr<parquet::encryption::CryptoFactory> crypto_factory_;
141-
inline static std::shared_ptr<parquet::encryption::KmsConnectionConfig>
142-
kms_connection_config_;
164+
std::shared_ptr<fs::FileSystem> file_system_;
165+
std::shared_ptr<Table> table_;
166+
std::shared_ptr<Partitioning> partitioning_;
167+
std::shared_ptr<parquet::encryption::CryptoFactory> crypto_factory_;
168+
std::shared_ptr<parquet::encryption::KmsConnectionConfig> kms_connection_config_;
169+
};
170+
171+
class DatasetEncryptionTest : public DatasetEncryptionTestBase {
172+
public:
173+
// The dataset is partitioned using a Hive partitioning scheme.
174+
void PrepareTableAndPartitioning() override {
175+
// Prepare table data.
176+
auto table_schema = schema({field("a", int64()), field("c", int64()),
177+
field("e", int64()), field("part", utf8())});
178+
table_ = TableFromJSON(table_schema, {R"([
179+
[ 0, 9, 1, "a" ],
180+
[ 1, 8, 2, "a" ],
181+
[ 2, 7, 1, "c" ],
182+
[ 3, 6, 2, "c" ],
183+
[ 4, 5, 1, "e" ],
184+
[ 5, 4, 2, "e" ],
185+
[ 6, 3, 1, "g" ],
186+
[ 7, 2, 2, "g" ],
187+
[ 8, 1, 1, "i" ],
188+
[ 9, 0, 2, "i" ]
189+
])"});
190+
191+
// Use a Hive-style partitioning scheme.
192+
partitioning_ = std::make_shared<HivePartitioning>(schema({field("part", utf8())}));
193+
}
143194
};
144195

145196
// This test demonstrates the process of writing a partitioned Parquet file with the same
@@ -148,44 +199,7 @@ class DatasetEncryptionTest : public ::testing::Test {
148199
// test reads the data back and verifies that it can be successfully decrypted and
149200
// scanned.
150201
TEST_F(DatasetEncryptionTest, WriteReadDatasetWithEncryption) {
151-
// Create decryption properties.
152-
auto decryption_config =
153-
std::make_shared<parquet::encryption::DecryptionConfiguration>();
154-
auto parquet_decryption_config = std::make_shared<ParquetDecryptionConfig>();
155-
parquet_decryption_config->crypto_factory = crypto_factory_;
156-
parquet_decryption_config->kms_connection_config = kms_connection_config_;
157-
parquet_decryption_config->decryption_config = std::move(decryption_config);
158-
159-
// Set scan options.
160-
auto parquet_scan_options = std::make_shared<ParquetFragmentScanOptions>();
161-
parquet_scan_options->parquet_decryption_config = std::move(parquet_decryption_config);
162-
163-
auto file_format = std::make_shared<ParquetFileFormat>();
164-
file_format->default_fragment_scan_options = std::move(parquet_scan_options);
165-
166-
// Get FileInfo objects for all files under the base directory
167-
fs::FileSelector selector;
168-
selector.base_dir = kBaseDir;
169-
selector.recursive = true;
170-
171-
FileSystemFactoryOptions factory_options;
172-
factory_options.partitioning = partitioning_;
173-
factory_options.partition_base_dir = kBaseDir;
174-
ASSERT_OK_AND_ASSIGN(auto dataset_factory,
175-
FileSystemDatasetFactory::Make(file_system_, selector, file_format,
176-
factory_options));
177-
178-
// Read dataset into table
179-
ASSERT_OK_AND_ASSIGN(auto dataset, dataset_factory->Finish());
180-
ASSERT_OK_AND_ASSIGN(auto scanner_builder, dataset->NewScan());
181-
ASSERT_OK_AND_ASSIGN(auto scanner, scanner_builder->Finish());
182-
ASSERT_OK_AND_ASSIGN(auto read_table, scanner->ToTable());
183-
184-
// Verify the data was read correctly
185-
ASSERT_OK_AND_ASSIGN(auto combined_table, read_table->CombineChunks());
186-
// Validate the table
187-
ASSERT_OK(combined_table->ValidateFull());
188-
AssertTablesEqual(*combined_table, *table_);
202+
ASSERT_NO_FATAL_FAILURE(TestScanDataset());
189203
}
190204

191205
// Read a single parquet file with and without decryption properties.
@@ -220,5 +234,26 @@ TEST_F(DatasetEncryptionTest, ReadSingleFile) {
220234
ASSERT_EQ(checked_pointer_cast<Int64Array>(table->column(2)->chunk(0))->GetView(0), 1);
221235
}
222236

237+
class LargeRowEncryptionTest : public DatasetEncryptionTestBase {
238+
public:
239+
// The dataset is partitioned using a Hive partitioning scheme.
240+
void PrepareTableAndPartitioning() override {
241+
// Create a random floating-point array with large number of rows.
242+
constexpr int kRowCount = 32769;
243+
arrow::random::RandomArrayGenerator rand_gen(0);
244+
auto array = rand_gen.Float32(kRowCount, 0.0, 1.0, false);
245+
auto table_schema = schema({field("a", float32())});
246+
247+
// Prepare table and partitioning.
248+
table_ = arrow::Table::Make(table_schema, {array});
249+
partitioning_ = std::make_shared<dataset::DirectoryPartitioning>(arrow::schema({}));
250+
}
251+
};
252+
253+
// Test for writing and reading encrypted dataset with large row count.
254+
TEST_F(LargeRowEncryptionTest, ReadEncryptLargeRows) {
255+
ASSERT_NO_FATAL_FAILURE(TestScanDataset());
256+
}
257+
223258
} // namespace dataset
224259
} // namespace arrow

0 commit comments

Comments
 (0)