Skip to content

Commit 5f57219

Browse files
Fokkolidavidmjorisvandenbossche
authored
GH-36845: [C++][Python] Allow type promotion on pa.concat_tables (#36846)
Revival of #12000 ### Rationale for this change It would be great to be able to do promotions when `concat`'ing a table, such as: ```python def test_concat_tables_with_promotion_int(): import pyarrow as pa t1 = pa.Table.from_arrays( [pa.array([1, 2], type=pa.int64())], ["int"]) t2 = pa.Table.from_arrays( [pa.array([3, 4], type=pa.int32())], ["int"]) result = pa.concat_tables([t1, t2], promote=True) assert result.equals(pa.Table.from_arrays([ pa.array([1, 2, 3, 4], type=pa.int64()) ], ["int"])) ``` ### What changes are included in this PR? ### Are these changes tested? ### Are there any user-facing changes? * Closes: #36845 Lead-authored-by: Fokko Driesprong <[email protected]> Co-authored-by: David Li <[email protected]> Co-authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent 0f7c569 commit 5f57219

17 files changed

Lines changed: 1220 additions & 71 deletions

File tree

‎c_glib/test/dataset/test-file-system-dataset-factory.rb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,9 @@ def test_validate_fragments
109109
point: Arrow::Int16DataType.new)
110110
options.validate_fragments = true
111111
message = "[file-system-dataset-factory][finish]: " +
112-
"Invalid: Unable to merge: " +
112+
"Type error: Unable to merge: " +
113113
"Field point has incompatible types: int16 vs int32"
114-
error = assert_raise(Arrow::Error::Invalid) do
114+
error = assert_raise(Arrow::Error::Type) do
115115
@factory.finish(options)
116116
end
117117
assert_equal(message, error.message.lines(chomp: true).first)

‎cpp/src/arrow/dataset/discovery.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ Result<std::shared_ptr<Schema>> DatasetFactory::Inspect(InspectOptions options)
4747
return arrow::schema({});
4848
}
4949

50-
return UnifySchemas(schemas);
50+
return UnifySchemas(schemas, options.field_merge_options);
5151
}
5252

5353
Result<std::shared_ptr<Dataset>> DatasetFactory::Finish() {

‎cpp/src/arrow/dataset/discovery.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ struct InspectOptions {
5858
/// `kInspectAllFragments`. A value of `0` disables inspection of fragments
5959
/// altogether so only the partitioning schema will be inspected.
6060
int fragments = 1;
61+
62+
/// Control how to unify types. By default, types are merged strictly (the
63+
/// type must match exactly, except nulls can be merged with other types).
64+
Field::MergeOptions field_merge_options = Field::MergeOptions::Defaults();
6165
};
6266

6367
struct FinishOptions {

‎cpp/src/arrow/dataset/discovery_test.cc‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,15 @@ TEST_F(MockDatasetFactoryTest, UnifySchemas) {
117117

118118
MakeFactory({schema({i32, f64}), schema({f64, i32_fake})});
119119
// Unification fails when fields with the same name have clashing types.
120-
ASSERT_RAISES(Invalid, factory_->Inspect());
120+
ASSERT_RAISES(TypeError, factory_->Inspect());
121121
// Return the individual schema for closer inspection should not fail.
122122
AssertInspectSchemas({schema({i32, f64}), schema({f64, i32_fake})});
123+
124+
MakeFactory({schema({field("num", int32())}), schema({field("num", float64())})});
125+
ASSERT_RAISES(TypeError, factory_->Inspect());
126+
InspectOptions permissive_options;
127+
permissive_options.field_merge_options = Field::MergeOptions::Permissive();
128+
AssertInspect(schema({field("num", float64())}), permissive_options);
123129
}
124130

125131
class FileSystemDatasetFactoryTest : public DatasetFactoryTest {
@@ -335,7 +341,7 @@ TEST_F(FileSystemDatasetFactoryTest, FinishWithIncompatibleSchemaShouldFail) {
335341
ASSERT_OK_AND_ASSIGN(auto dataset, factory_->Finish(options));
336342

337343
MakeFactory({fs::File("test")});
338-
ASSERT_RAISES(Invalid, factory_->Finish(options));
344+
ASSERT_RAISES(TypeError, factory_->Finish(options));
339345

340346
// Disable validation
341347
options.validate_fragments = false;
@@ -463,8 +469,8 @@ TEST(UnionDatasetFactoryTest, ConflictingSchemas) {
463469
{dataset_factory_1, dataset_factory_2, dataset_factory_3}));
464470

465471
// schema_3 conflicts with other, Inspect/Finish should not work
466-
ASSERT_RAISES(Invalid, factory->Inspect());
467-
ASSERT_RAISES(Invalid, factory->Finish());
472+
ASSERT_RAISES(TypeError, factory->Inspect());
473+
ASSERT_RAISES(TypeError, factory->Finish());
468474

469475
// The user can inspect without error
470476
ASSERT_OK_AND_ASSIGN(auto schemas, factory->InspectSchemas({}));
@@ -474,6 +480,12 @@ TEST(UnionDatasetFactoryTest, ConflictingSchemas) {
474480
auto i32_schema = schema({i32});
475481
ASSERT_OK_AND_ASSIGN(auto dataset, factory->Finish(i32_schema));
476482
EXPECT_EQ(*dataset->schema(), *i32_schema);
483+
484+
// The user decided to allow merging the types.
485+
FinishOptions options;
486+
options.inspect_options.field_merge_options = Field::MergeOptions::Permissive();
487+
ASSERT_OK_AND_ASSIGN(dataset, factory->Finish(options));
488+
EXPECT_EQ(*dataset->schema(), *schema({f64, i32}));
477489
}
478490

479491
} // namespace dataset

‎cpp/src/arrow/table.cc‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "arrow/array/concatenate.h"
3131
#include "arrow/array/util.h"
3232
#include "arrow/chunked_array.h"
33+
#include "arrow/compute/cast.h"
3334
#include "arrow/pretty_print.h"
3435
#include "arrow/record_batch.h"
3536
#include "arrow/result.h"
@@ -450,6 +451,13 @@ Result<std::shared_ptr<Table>> ConcatenateTables(
450451
Result<std::shared_ptr<Table>> PromoteTableToSchema(const std::shared_ptr<Table>& table,
451452
const std::shared_ptr<Schema>& schema,
452453
MemoryPool* pool) {
454+
return PromoteTableToSchema(table, schema, compute::CastOptions::Safe(), pool);
455+
}
456+
457+
Result<std::shared_ptr<Table>> PromoteTableToSchema(const std::shared_ptr<Table>& table,
458+
const std::shared_ptr<Schema>& schema,
459+
const compute::CastOptions& options,
460+
MemoryPool* pool) {
453461
const std::shared_ptr<Schema> current_schema = table->schema();
454462
if (current_schema->Equals(*schema, /*check_metadata=*/false)) {
455463
return table->ReplaceSchemaMetadata(schema->metadata());
@@ -487,8 +495,8 @@ Result<std::shared_ptr<Table>> PromoteTableToSchema(const std::shared_ptr<Table>
487495
const int field_index = field_indices[0];
488496
const auto& current_field = current_schema->field(field_index);
489497
if (!field->nullable() && current_field->nullable()) {
490-
return Status::Invalid("Unable to promote field ", current_field->name(),
491-
": it was nullable but the target schema was not.");
498+
return Status::TypeError("Unable to promote field ", current_field->name(),
499+
": it was nullable but the target schema was not.");
492500
}
493501

494502
fields_seen[field_index] = true;
@@ -502,9 +510,15 @@ Result<std::shared_ptr<Table>> PromoteTableToSchema(const std::shared_ptr<Table>
502510
continue;
503511
}
504512

505-
return Status::Invalid("Unable to promote field ", field->name(),
506-
": incompatible types: ", field->type()->ToString(), " vs ",
507-
current_field->type()->ToString());
513+
if (!compute::CanCast(*current_field->type(), *field->type())) {
514+
return Status::TypeError("Unable to promote field ", field->name(),
515+
": incompatible types: ", field->type()->ToString(),
516+
" vs ", current_field->type()->ToString());
517+
}
518+
compute::ExecContext ctx(pool);
519+
ARROW_ASSIGN_OR_RAISE(auto casted, compute::Cast(table->column(field_index),
520+
field->type(), options, &ctx));
521+
columns.push_back(casted.chunked_array());
508522
}
509523

510524
auto unseen_field_iter = std::find(fields_seen.begin(), fields_seen.end(), false);

‎cpp/src/arrow/table.h‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -313,16 +313,23 @@ Result<std::shared_ptr<Table>> ConcatenateTables(
313313
ConcatenateTablesOptions options = ConcatenateTablesOptions::Defaults(),
314314
MemoryPool* memory_pool = default_memory_pool());
315315

316+
namespace compute {
317+
class CastOptions;
318+
}
319+
316320
/// \brief Promotes a table to conform to the given schema.
317321
///
318-
/// If a field in the schema does not have a corresponding column in the
319-
/// table, a column of nulls will be added to the resulting table.
320-
/// If the corresponding column is of type Null, it will be promoted to
321-
/// the type specified by schema, with null values filled.
322+
/// If a field in the schema does not have a corresponding column in
323+
/// the table, a column of nulls will be added to the resulting table.
324+
/// If the corresponding column is of type Null, it will be promoted
325+
/// to the type specified by schema, with null values filled. The
326+
/// column will be casted to the type specified by the schema.
327+
///
322328
/// Returns an error:
323329
/// - if the corresponding column's type is not compatible with the
324330
/// schema.
325331
/// - if there is a column in the table that does not exist in the schema.
332+
/// - if the cast fails or casting would be required but is not available.
326333
///
327334
/// \param[in] table the input Table
328335
/// \param[in] schema the target schema to promote to
@@ -333,4 +340,28 @@ Result<std::shared_ptr<Table>> PromoteTableToSchema(
333340
const std::shared_ptr<Table>& table, const std::shared_ptr<Schema>& schema,
334341
MemoryPool* pool = default_memory_pool());
335342

343+
/// \brief Promotes a table to conform to the given schema.
344+
///
345+
/// If a field in the schema does not have a corresponding column in
346+
/// the table, a column of nulls will be added to the resulting table.
347+
/// If the corresponding column is of type Null, it will be promoted
348+
/// to the type specified by schema, with null values filled. The column
349+
/// will be casted to the type specified by the schema.
350+
///
351+
/// Returns an error:
352+
/// - if the corresponding column's type is not compatible with the
353+
/// schema.
354+
/// - if there is a column in the table that does not exist in the schema.
355+
/// - if the cast fails or casting would be required but is not available.
356+
///
357+
/// \param[in] table the input Table
358+
/// \param[in] schema the target schema to promote to
359+
/// \param[in] options The cast options to allow promotion of types
360+
/// \param[in] pool The memory pool to be used if null-filled arrays need to
361+
/// be created.
362+
ARROW_EXPORT
363+
Result<std::shared_ptr<Table>> PromoteTableToSchema(
364+
const std::shared_ptr<Table>& table, const std::shared_ptr<Schema>& schema,
365+
const compute::CastOptions& options, MemoryPool* pool = default_memory_pool());
366+
336367
} // namespace arrow

‎cpp/src/arrow/table_test.cc‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include "arrow/array/data.h"
3030
#include "arrow/array/util.h"
3131
#include "arrow/chunked_array.h"
32+
#include "arrow/compute/cast.h"
3233
#include "arrow/record_batch.h"
3334
#include "arrow/status.h"
3435
#include "arrow/testing/gtest_util.h"
@@ -418,16 +419,17 @@ TEST_F(TestPromoteTableToSchema, IncompatibleTypes) {
418419
auto table = MakeTableWithOneNullFilledColumn("field", int32(), length);
419420

420421
// Invalid promotion: int32 to null.
421-
ASSERT_RAISES(Invalid, PromoteTableToSchema(table, schema({field("field", null())})));
422+
ASSERT_RAISES(TypeError, PromoteTableToSchema(table, schema({field("field", null())})));
422423

423-
// Invalid promotion: int32 to uint32.
424-
ASSERT_RAISES(Invalid, PromoteTableToSchema(table, schema({field("field", uint32())})));
424+
// Invalid promotion: int32 to list.
425+
ASSERT_RAISES(TypeError,
426+
PromoteTableToSchema(table, schema({field("field", list(int32()))})));
425427
}
426428

427429
TEST_F(TestPromoteTableToSchema, IncompatibleNullity) {
428430
const int length = 10;
429431
auto table = MakeTableWithOneNullFilledColumn("field", int32(), length);
430-
ASSERT_RAISES(Invalid,
432+
ASSERT_RAISES(TypeError,
431433
PromoteTableToSchema(
432434
table, schema({field("field", uint32())->WithNullable(false)})));
433435
}
@@ -520,6 +522,36 @@ TEST_F(ConcatenateTablesWithPromotionTest, Simple) {
520522
AssertTablesEqualUnorderedFields(*expected, *result);
521523
}
522524

525+
TEST_F(ConcatenateTablesWithPromotionTest, Unify) {
526+
auto t_i32 = TableFromJSON(schema({field("f0", int32())}), {"[[0], [1]]"});
527+
auto t_i64 = TableFromJSON(schema({field("f0", int64())}), {"[[2], [3]]"});
528+
auto t_null = TableFromJSON(schema({field("f0", null())}), {"[[null], [null]]"});
529+
530+
auto expected_int64 =
531+
TableFromJSON(schema({field("f0", int64())}), {"[[0], [1], [2], [3]]"});
532+
auto expected_null =
533+
TableFromJSON(schema({field("f0", int32())}), {"[[0], [1], [null], [null]]"});
534+
535+
ConcatenateTablesOptions options;
536+
EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid,
537+
::testing::HasSubstr("Schema at index 1 was different"),
538+
ConcatenateTables({t_i32, t_i64}, options));
539+
EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid,
540+
::testing::HasSubstr("Schema at index 1 was different"),
541+
ConcatenateTables({t_i32, t_null}, options));
542+
543+
options.unify_schemas = true;
544+
EXPECT_RAISES_WITH_MESSAGE_THAT(TypeError,
545+
::testing::HasSubstr("Field f0 has incompatible types"),
546+
ConcatenateTables({t_i64, t_i32}, options));
547+
ASSERT_OK_AND_ASSIGN(auto actual, ConcatenateTables({t_i32, t_null}, options));
548+
AssertTablesEqual(*expected_null, *actual, /*same_chunk_layout=*/false);
549+
550+
options.field_merge_options.promote_numeric_width = true;
551+
ASSERT_OK_AND_ASSIGN(actual, ConcatenateTables({t_i32, t_i64}, options));
552+
AssertTablesEqual(*expected_int64, *actual, /*same_chunk_layout=*/false);
553+
}
554+
523555
TEST_F(TestTable, Slice) {
524556
const int64_t length = 10;
525557

0 commit comments

Comments
 (0)